From 08c3e9fd5735ff395b4931777dac0fca6a375610 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 20 Nov 2012 22:40:48 +0000 Subject: [PATCH] Basic set of listing and upload commands --- notes.txt | 8 +++ swiftsync.go | 183 +++++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 171 insertions(+), 20 deletions(-) diff --git a/notes.txt b/notes.txt index 4878cade4..5c1cea768 100644 --- a/notes.txt +++ b/notes.txt @@ -30,3 +30,11 @@ https://github.com/gholt/swiftly As an integer, but it does parse it as a float subargs.append('x-object-meta-mtime:%d' % getmtime(options.input_)) + +Need an iterate all objects routine... Could use a channel +- could just be a flag to Objects() + +FIXME ordering not hash order + +FIXME progress meter would be nice! Do this by wrapping the Reader with a progress bar + diff --git a/swiftsync.go b/swiftsync.go index 4be927521..dbd6744fc 100644 --- a/swiftsync.go +++ b/swiftsync.go @@ -4,25 +4,15 @@ package main import ( - //"bytes" "flag" "fmt" - //"io" - //"io/ioutil" + "github.com/ncw/swift" "log" - //"math/rand" "os" - //"os/signal" - //"path/filepath" - //"regexp" - //"runtime" + "path/filepath" "runtime/pprof" "strconv" "strings" - //"sync" - //"syscall" - //"time" - "github.com/ncw/swift" ) // Globals @@ -43,6 +33,74 @@ var ( apiKey = flag.String("key", os.Getenv("ST_KEY"), "API key (password). Defaults to environment var ST_KEY.") ) +type FsObject struct { + rel string + path string + info os.FileInfo +} + +type FsObjects map[string]FsObject + +// Puts the FsObject into the container +func (fs *FsObject) put(c *swift.Connection, container string) { + mode := fs.info.Mode() + if mode&(os.ModeSymlink|os.ModeNamedPipe|os.ModeSocket|os.ModeDevice) != 0 { + log.Printf("Can't transfer non file/directory %s", fs.path) + } else if mode&os.ModeDir != 0 { + // Debug? + log.Printf("FIXME Didn't upload %s", fs.path) + } else { + // FIXME content type + // FIXME headers with mtime in + in, err := os.Open(fs.path) + if err != nil { + log.Printf("Failed to open %s: %s", fs.path, err) + return + } + defer in.Close() + _, err = c.ObjectPut(container, fs.rel, in, true, "", "", nil) + if err != nil { + log.Printf("Failed to upload %s: %s", fs.path, err) + return + } + log.Printf("Uploaded %s", fs.path) + } + +} + +// Walk the path +// +// FIXME ignore symlinks? +// FIXME what about hardlinks / etc +func walk(root string) FsObjects { + files := make(FsObjects) + err := filepath.Walk(root, func(path string, f os.FileInfo, err error) error { + if err != nil { + log.Printf("Failed to open directory: %s: %s", path, err) + } else { + info, err := os.Stat(path) + if err != nil { + log.Printf("Failed to stat %s: %s", path, err) + return nil + } + rel, err := filepath.Rel(root, path) + if err != nil { + log.Printf("Failed to get relative path %s: %s", path, err) + return nil + } + if rel == "." { + rel = "" + } + files[rel] = FsObject{rel: rel, path: path, info: info} + } + return nil + }) + if err != nil { + log.Printf("Failed to open directory: %s: %s", root, err) + } + return files +} + // Turns a number of ns into a floating point string in seconds // // Trims trailing zeros and guaranteed to be perfectly accurate @@ -113,10 +171,70 @@ func fatal(message string, args ...interface{}) { os.Exit(1) } +// checkArgs checks there are enough arguments and prints a message if not +func checkArgs(args []string, n int, message string) { + if len(args) != n { + syntaxError() + fmt.Fprintf(os.Stderr, "%d arguments required: %s\n", n, message) + os.Exit(1) + } +} + +// uploads a file into a container +func upload(c *swift.Connection, root, container string) { + files := walk(root) + for _, fs := range files { + fs.put(c, container) + } +} + +// Lists the containers +func listContainers(c *swift.Connection) { + containers, err := c.Containers(nil) + if err != nil { + log.Fatalf("Couldn't list containers: %s", err) + } + for _, container := range containers { + fmt.Printf("%9d %12d %s\n", container.Count, container.Bytes, container.Name) + } +} + +// Lists files in a container +func list(c *swift.Connection, container string) { + //objects, err := c.Objects(container, &swift.ObjectsOpts{Prefix: "", Delimiter: '/'}) + objects, err := c.Objects(container, nil) + if err != nil { + log.Fatalf("Couldn't read container %q: %s", container, err) + } + for _, object := range objects { + if object.PseudoDirectory { + fmt.Printf("%9s %19s %s\n", "Directory", "-", object.Name) + } else { + fmt.Printf("%9d %19s %s\n", object.Bytes, object.LastModified.Format("2006-01-02 15:04:05"), object.Name) + } + } +} + +// Makes a container +func mkdir(c *swift.Connection, container string) { + err := c.ContainerCreate(container, nil) + if err != nil { + log.Fatalf("Couldn't create container %q: %s", container, err) + } +} + +// Removes a container +func rmdir(c *swift.Connection, container string) { + err := c.ContainerDelete(container) + if err != nil { + log.Fatalf("Couldn't delete container %q: %s", container, err) + } +} + func main() { flag.Usage = syntaxError flag.Parse() - //args := flag.Args() + args := flag.Args() //runtime.GOMAXPROCS(3) // Setup profiling if desired @@ -129,20 +247,21 @@ func main() { defer pprof.StopCPUProfile() } - // if len(args) < 1 { - // fatal("No command supplied\n") - // } + fmt.Println(args) + if len(args) < 1 { + fatal("No command supplied\n") + } if *userName == "" { - log.Fatal("Need --user or environmental variable ST_USER") + log.Fatal("Need -user or environmental variable ST_USER") } if *apiKey == "" { - log.Fatal("Need --key or environmental variable ST_KEY") + log.Fatal("Need -key or environmental variable ST_KEY") } if *authUrl == "" { - log.Fatal("Need --auth or environmental variable ST_AUTH") + log.Fatal("Need -auth or environmental variable ST_AUTH") } - c := swift.Connection{ + c := &swift.Connection{ UserName: *userName, ApiKey: *apiKey, AuthUrl: *authUrl, @@ -152,4 +271,28 @@ func main() { log.Fatal("Failed to authenticate", err) } + command := args[0] + args = args[1:] + + switch command { + case "up", "upload": + checkArgs(args, 2, "Need directory to read from and container to write to") + upload(c, args[0], args[1]) + case "list", "ls": + if len(args) == 0 { + listContainers(c) + } else { + checkArgs(args, 1, "Need container to list") + list(c, args[0]) + } + case "mkdir": + checkArgs(args, 1, "Need container to make") + mkdir(c, args[0]) + case "rmdir": + checkArgs(args, 1, "Need container to delte") + rmdir(c, args[0]) + default: + log.Fatalf("Unknown command %q", command) + } + }