diff --git a/fs.go b/fs.go index 7b167e526..f51b6b233 100644 --- a/fs.go +++ b/fs.go @@ -17,6 +17,9 @@ type Fs interface { // List the Fs into a channel List() FsObjectsChan + // List the Fs directories/buckets/containers into a channel + ListDir() FsDirChan + // Find the FsObject at remote. Returns nil if can't be found NewFsObject(remote string) FsObject @@ -82,6 +85,17 @@ type FsObjectsChan chan FsObject // A slice of FsObjects type FsObjects []FsObject +// A structure of directory/container/bucket lists +type FsDir struct { + Name string // name of the directory + When time.Time // modification or creation time - IsZero for unknown + Bytes int64 // size of directory and contents -1 for unknown + Count int64 // number of objects -1 for unknown +} + +// A channel of FsDir objects +type FsDirChan chan *FsDir + // NewFs makes a new Fs object from the path // // FIXME make more generic diff --git a/fs_drive.go b/fs_drive.go index 8dfe97a9f..965db9981 100644 --- a/fs_drive.go +++ b/fs_drive.go @@ -568,6 +568,35 @@ func (f *FsDrive) List() FsObjectsChan { return out } +// Walk the path returning a channel of FsObjects +func (f *FsDrive) ListDir() FsDirChan { + out := make(FsDirChan, *checkers) + go func() { + defer close(out) + err := f.findRoot(false) + if err != nil { + stats.Error() + log.Printf("Couldn't find root: %s", err) + } else { + _, err := f.listAll(f.rootId, "", true, false, func(item *drive.File) bool { + dir := &FsDir{ + Name: item.Title, + Bytes: -1, + Count: -1, + } + dir.When, _ = time.Parse(time.RFC3339, item.ModifiedDate) + out <- dir + return false + }) + if err != nil { + stats.Error() + log.Printf("ListDir failed: %s", err) + } + } + }() + return out +} + // Put the FsObject into the container // // Copy the reader in to the new object which is returned diff --git a/fs_local.go b/fs_local.go index 33036fd5d..600a20444 100644 --- a/fs_local.go +++ b/fs_local.go @@ -105,6 +105,49 @@ func (f *FsLocal) List() FsObjectsChan { return out } +// Walk the path returning a channel of FsObjects +func (f *FsLocal) ListDir() FsDirChan { + out := make(FsDirChan, *checkers) + go func() { + defer close(out) + items, err := ioutil.ReadDir(f.root) + if err != nil { + stats.Error() + log.Printf("Couldn't find read directory: %s", err) + } else { + for _, item := range items { + if item.IsDir() { + dir := &FsDir{ + Name: item.Name(), + When: item.ModTime(), + Bytes: 0, + Count: 0, + } + // Go down the tree to count the files and directories + dirpath := path.Join(f.root, item.Name()) + err := filepath.Walk(dirpath, func(path string, fi os.FileInfo, err error) error { + if err != nil { + stats.Error() + log.Printf("Failed to open directory: %s: %s", path, err) + } else { + dir.Count += 1 + dir.Bytes += fi.Size() + } + return nil + }) + if err != nil { + stats.Error() + log.Printf("Failed to open directory: %s: %s", dirpath, err) + } + out <- dir + } + } + } + // err := f.findRoot(false) + }() + return out +} + // Puts the FsObject to the local filesystem func (f *FsLocal) Put(in io.Reader, remote string, modTime time.Time, size int64) (FsObject, error) { dstPath := filepath.Join(f.root, remote) diff --git a/fs_s3.go b/fs_s3.go index c9ef02607..916de500f 100644 --- a/fs_s3.go +++ b/fs_s3.go @@ -1,6 +1,8 @@ // S3 interface package main +// FIXME need to prevent anything but ListDir working for s3:// + import ( "errors" "flag" @@ -65,7 +67,7 @@ func (f *FsS3) String() string { } // Pattern to match a s3 url -var s3Match = regexp.MustCompile(`^s3://([^/]+)(.*)$`) +var s3Match = regexp.MustCompile(`^s3://([^/]*)(.*)$`) // parseParse parses a s3 'url' func s3ParsePath(path string) (bucket, directory string, err error) { @@ -132,23 +134,6 @@ func NewFsS3(path string) (*FsS3, error) { return f, nil } -// Lists the buckets -func S3Buckets() { - c, err := s3Connection() - if err != nil { - stats.Error() - log.Fatalf("Couldn't connect: %s", err) - } - buckets, err := c.List() - if err != nil { - stats.Error() - log.Fatalf("Couldn't list buckets: %s", err) - } - for _, bucket := range buckets.Buckets { - fmt.Printf("%12s %s\n", bucket.CreationDate, bucket.Name) - } -} - // Return an FsObject from a path // // May return nil if an error occurred @@ -206,6 +191,30 @@ func (f *FsS3) List() FsObjectsChan { return out } +// Lists the buckets +func (f *FsS3) ListDir() FsDirChan { + out := make(FsDirChan, *checkers) + go func() { + defer close(out) + buckets, err := f.c.List() + if err != nil { + stats.Error() + log.Printf("Couldn't list buckets: %s", err) + } else { + for _, bucket := range buckets.Buckets { + when, _ := time.Parse(time.RFC3339, bucket.CreationDate) + out <- &FsDir{ + Name: bucket.Name, + When: when, + Bytes: -1, + Count: -1, + } + } + } + }() + return out +} + // Put the FsObject into the bucket func (f *FsS3) Put(in io.Reader, remote string, modTime time.Time, size int64) (FsObject, error) { // Temporary FsObject under construction diff --git a/fs_swift.go b/fs_swift.go index 8db951d36..5cf66645a 100644 --- a/fs_swift.go +++ b/fs_swift.go @@ -1,6 +1,8 @@ // Swift interface package main +// FIXME need to prevent anything but ListDir working for swift:// + import ( "errors" "flag" @@ -48,7 +50,7 @@ func (f *FsSwift) String() string { } // Pattern to match a swift url -var swiftMatch = regexp.MustCompile(`^swift://([^/]+)(.*)$`) +var swiftMatch = regexp.MustCompile(`^swift://([^/]*)(.*)$`) // parseParse parses a swift 'url' func parsePath(path string) (container, directory string, err error) { @@ -102,23 +104,6 @@ func NewFsSwift(path string) (*FsSwift, error) { return f, nil } -// Lists the containers -func SwiftContainers() { - c, err := swiftConnection() - if err != nil { - stats.Error() - log.Fatalf("Couldn't connect: %s", err) - } - containers, err := c.ContainersAll(nil) - if err != nil { - stats.Error() - log.Fatalf("Couldn't list containers: %s", err) - } - for _, container := range containers { - fmt.Printf("%9d %12d %s\n", container.Count, container.Bytes, container.Name) - } -} - // Return an FsObject from a path // // May return nil if an error occurred @@ -173,6 +158,28 @@ func (f *FsSwift) List() FsObjectsChan { return out } +// Lists the containers +func (f *FsSwift) ListDir() FsDirChan { + out := make(FsDirChan, *checkers) + go func() { + defer close(out) + containers, err := f.c.ContainersAll(nil) + if err != nil { + stats.Error() + log.Printf("Couldn't list containers: %s", err) + } else { + for _, container := range containers { + out <- &FsDir{ + Name: container.Name, + Bytes: container.Bytes, + Count: container.Count, + } + } + } + }() + return out +} + // Put the FsObject into the container // // Copy the reader in to the new object which is returned diff --git a/notes.txt b/notes.txt index 101a37d64..5c9d7443e 100644 --- a/notes.txt +++ b/notes.txt @@ -1,3 +1,19 @@ +Names + * cakesync + * zzsync + * pqrsync + * gogosync + * pogosync + * oursync + * godupe + * gobsync + * gobcopy + * togosync + * tangosync + * mangosync + * cargosync + * cargocopy - domain available + Todo * Make a test suite which can run on all the given types of fs * Copy should use the sync code as it is more efficient at directory listing diff --git a/swiftsync.go b/swiftsync.go index dafb2adff..dc9326647 100644 --- a/swiftsync.go +++ b/swiftsync.go @@ -300,7 +300,7 @@ func Check(fdst, fsrc Fs) { // List the Fs to stdout // // Lists in parallel which may get them out of order -func List(f Fs) { +func List(f, _ Fs) { in := f.List() var wg sync.WaitGroup wg.Add(*checkers) @@ -318,14 +318,11 @@ func List(f Fs) { wg.Wait() } -// Lists files in a container -func list(fdst, fsrc Fs) { - if fdst == nil { - // FIXMESwiftContainers() - S3Buckets() - return +// List the directories/buckets/containers in the Fs to stdout +func ListDir(f, _ Fs) { + for dir := range f.ListDir() { + fmt.Printf("%12d %13s %9d %s\n", dir.Bytes, dir.When.Format("2006-01-02 15:04:05"), dir.Count, dir.Name) } - List(fdst) } // Makes a destination directory or container @@ -417,11 +414,19 @@ var Commands = []Command{ "ls", `[] - List the path. If no parameter is supplied then it lists the - available swift containers.`, + List all the objects in the the path.`, - list, - 0, 1, + List, + 1, 1, + }, + { + "lsd", + `[] + + List all directoryes/objects/buckets in the the path.`, + + ListDir, + 1, 1, }, { "mkdir",