From 215fd2a11dd0ab7167cfc58389b6d7b8a8ab518f Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 14 Dec 2016 17:37:26 +0000 Subject: [PATCH] b2: use new prefix and delimiter parameters in directory listings This makes --max-depth 1 directory listings much more efficient (it no longer lists all the files) and simplifies the code, bringing it into line with s3/swift/gcs Fixes #944 --- b2/api/types.go | 2 + b2/b2.go | 81 ++++++++++------------------- b2/b2_internal_test.go | 114 ----------------------------------------- 3 files changed, 30 insertions(+), 167 deletions(-) diff --git a/b2/api/types.go b/b2/api/types.go index 05acc6c27..bbb4e83b8 100644 --- a/b2/api/types.go +++ b/b2/api/types.go @@ -154,6 +154,8 @@ type ListFileNamesRequest struct { StartFileName string `json:"startFileName,omitempty"` // optional - The first file name to return. If there is a file with this name, it will be returned in the list. If not, the first file name after this the first one after this name. MaxFileCount int `json:"maxFileCount,omitempty"` // optional - The maximum number of files to return from this call. The default value is 100, and the maximum allowed is 1000. StartFileID string `json:"startFileId,omitempty"` // optional - What to pass in to startFileId for the next search to continue where this one left off. + Prefix string `json:"prefix,omitempty"` // optional - Files returned will be limited to those with the given prefix. Defaults to the empty string, which matches all files. + Delimiter string `json:"delimiter,omitempty"` // Files returned will be limited to those within the top folder, or any one subfolder. Defaults to NULL. Folder names will also be returned. The delimiter character will be used to "break" file names into folders. } // ListFileNamesResponse is as received from b2_list_file_names or b2_list_file_versions diff --git a/b2/b2.go b/b2/b2.go index 149ab777b..d2bef5f47 100644 --- a/b2/b2.go +++ b/b2/b2.go @@ -465,34 +465,6 @@ func (f *Fs) NewObject(remote string) (fs.Object, error) { return f.newObjectWithInfo(remote, nil) } -// sendDir works out given a lastDir and a remote which directories should be sent -func sendDir(lastDir string, remote string, level int) (dirNames []string, newLastDir string) { - dir := path.Dir(remote) - if dir == "." { - // No slashes - nothing to do! - return nil, lastDir - } - if dir == lastDir { - // Still in same directory - return nil, lastDir - } - newLastDir = lastDir - for { - slashes := strings.Count(dir, "/") - if !strings.HasPrefix(lastDir, dir) && slashes < level { - dirNames = append([]string{dir}, dirNames...) - } - if newLastDir == lastDir { - newLastDir = dir - } - dir = path.Dir(dir) - if dir == "." { - break - } - } - return dirNames, newLastDir -} - // listFn is called from list to handle an object type listFn func(remote string, object *api.File, isDirectory bool) error @@ -503,6 +475,8 @@ var errEndList = errors.New("end list") // list lists the objects into the function supplied from // the bucket and root supplied // +// dir is the starting directory, "" for root +// // level is the depth to search to // // If prefix is set then startFileName is used as a prefix which all @@ -517,6 +491,14 @@ func (f *Fs) list(dir string, level int, prefix string, limit int, hidden bool, if dir != "" { root += dir + "/" } + delimiter := "" + switch level { + case 1: + delimiter = "/" + case fs.MaxLevel: + default: + return fs.ErrorLevelNotSupported + } bucketID, err := f.getBucketID() if err != nil { return err @@ -528,6 +510,8 @@ func (f *Fs) list(dir string, level int, prefix string, limit int, hidden bool, var request = api.ListFileNamesRequest{ BucketID: bucketID, MaxFileCount: chunkSize, + Prefix: root, + Delimiter: delimiter, } prefix = root + prefix if prefix != "" { @@ -541,7 +525,6 @@ func (f *Fs) list(dir string, level int, prefix string, limit int, hidden bool, if hidden { opts.Path = "/b2_list_file_versions" } - lastDir := dir for { err := f.pacer.Call(func() (bool, error) { resp, err := f.srv.CallJSON(&opts, &request, &response) @@ -553,34 +536,26 @@ func (f *Fs) list(dir string, level int, prefix string, limit int, hidden bool, for i := range response.Files { file := &response.Files[i] // Finish if file name no longer has prefix - if !strings.HasPrefix(file.Name, prefix) { + if prefix != "" && !strings.HasPrefix(file.Name, prefix) { return nil } - remote := file.Name[len(f.root):] - slashes := strings.Count(remote, "/") - - // Check if this file makes a new directories - var dirNames []string - dirNames, lastDir = sendDir(lastDir, remote, level) - for _, dirName := range dirNames { - err = fn(dirName, nil, true) - if err != nil { - if err == errEndList { - return nil - } - return err - } + if !strings.HasPrefix(file.Name, f.root) { + fs.Log(f, "Odd name received %q", file.Name) + continue } - - // Send the file - if slashes < level { - err = fn(remote, file, false) - if err != nil { - if err == errEndList { - return nil - } - return err + remote := file.Name[len(f.root):] + // Check for directory + isDirectory := level != 0 && strings.HasSuffix(remote, "/") + if isDirectory { + remote = remote[:len(remote)-1] + } + // Send object + err = fn(remote, file, isDirectory) + if err != nil { + if err == errEndList { + return nil } + return err } } // end if no NextFileName diff --git a/b2/b2_internal_test.go b/b2/b2_internal_test.go index 25ac2f1b2..f64e83bcb 100644 --- a/b2/b2_internal_test.go +++ b/b2/b2_internal_test.go @@ -5,7 +5,6 @@ import ( "time" "github.com/ncw/rclone/fstest" - "github.com/stretchr/testify/assert" ) // Test b2 string encoding @@ -169,116 +168,3 @@ func TestParseTimeString(t *testing.T) { } } - -func TestSendDir(t *testing.T) { - for _, test := range []struct { - lastDir string - remote string - level int - dirNames []string - newLastDir string - }{ - { - lastDir: "", - remote: "test.txt", - level: 100, - dirNames: nil, - newLastDir: "", - }, - { - lastDir: "", - remote: "potato/test.txt", - level: 100, - dirNames: []string{"potato"}, - newLastDir: "potato", - }, - { - lastDir: "potato", - remote: "potato/test.txt", - level: 100, - dirNames: nil, - newLastDir: "potato", - }, - { - lastDir: "", - remote: "potato/sausage/test.txt", - level: 100, - dirNames: []string{"potato", "potato/sausage"}, - newLastDir: "potato/sausage", - }, - { - lastDir: "potato", - remote: "potato/sausage/test.txt", - level: 100, - dirNames: []string{"potato/sausage"}, - newLastDir: "potato/sausage", - }, - { - lastDir: "potato/sausage", - remote: "potato/sausage/test.txt", - level: 100, - dirNames: nil, - newLastDir: "potato/sausage", - }, - { - lastDir: "", - remote: "a/b/c/d/e/f.txt", - level: 100, - dirNames: []string{"a", "a/b", "a/b/c", "a/b/c/d", "a/b/c/d/e"}, - newLastDir: "a/b/c/d/e", - }, - { - lastDir: "a/b/c/d/e", - remote: "a/b/c/d/E/f.txt", - level: 100, - dirNames: []string{"a/b/c/d/E"}, - newLastDir: "a/b/c/d/E", - }, - { - lastDir: "a/b/c/d/e", - remote: "a/b/C/D/E/f.txt", - level: 100, - dirNames: []string{"a/b/C", "a/b/C/D", "a/b/C/D/E"}, - newLastDir: "a/b/C/D/E", - }, - { - lastDir: "a/b/c", - remote: "a/b/c/d/e/f.txt", - level: 100, - dirNames: []string{"a/b/c/d", "a/b/c/d/e"}, - newLastDir: "a/b/c/d/e", - }, - { - lastDir: "", - remote: "a/b/c/d/e/f.txt", - level: 1, - dirNames: []string{"a"}, - newLastDir: "a/b/c/d/e", - }, - { - lastDir: "a/b/c", - remote: "a/b/c/d/e/f.txt", - level: 1, - dirNames: nil, - newLastDir: "a/b/c/d/e", - }, - { - lastDir: "", - remote: "a/b/c/d/e/f.txt", - level: 3, - dirNames: []string{"a", "a/b", "a/b/c"}, - newLastDir: "a/b/c/d/e", - }, - { - lastDir: "a/b/C/D/E", - remote: "a/b/c/d/e/f.txt", - level: 3, - dirNames: []string{"a/b/c"}, - newLastDir: "a/b/c/d/e", - }, - } { - dirNames, newLastDir := sendDir(test.lastDir, test.remote, test.level) - assert.Equal(t, test.dirNames, dirNames, "dirNames fail for sendDir(%q,%q,%v)", test.lastDir, test.remote, test.level) - assert.Equal(t, test.newLastDir, newLastDir, "newLastDir fail for sendDir(%q,%q,%v)", test.lastDir, test.remote, test.level) - } -}