b2: use new prefix and delimiter parameters in directory listings

This makes --max-depth 1 directory listings much more efficient (it no
longer lists all the files) and simplifies the code, bringing it into
line with s3/swift/gcs

Fixes #944

b2/api/types.go

@@ -154,6 +154,8 @@ type ListFileNamesRequest struct {
 	StartFileName string `json:"startFileName,omitempty"` // optional - The first file name to return. If there is a file with this name, it will be returned in the list. If not, the first file name after this the first one after this name.
 	MaxFileCount  int    `json:"maxFileCount,omitempty"`  // optional - The maximum number of files to return from this call. The default value is 100, and the maximum allowed is 1000.
 	StartFileID   string `json:"startFileId,omitempty"`   // optional - What to pass in to startFileId for the next search to continue where this one left off.
+	Prefix        string `json:"prefix,omitempty"`        // optional - Files returned will be limited to those with the given prefix. Defaults to the empty string, which matches all files.
+	Delimiter     string `json:"delimiter,omitempty"`     // Files returned will be limited to those within the top folder, or any one subfolder. Defaults to NULL. Folder names will also be returned. The delimiter character will be used to "break" file names into folders.
 }
 
 // ListFileNamesResponse is as received from b2_list_file_names or b2_list_file_versions

b2/b2.go

@@ -465,34 +465,6 @@ func (f *Fs) NewObject(remote string) (fs.Object, error) {
 	return f.newObjectWithInfo(remote, nil)
 }
 
-// sendDir works out given a lastDir and a remote which directories should be sent
-func sendDir(lastDir string, remote string, level int) (dirNames []string, newLastDir string) {
-	dir := path.Dir(remote)
-	if dir == "." {
-		// No slashes - nothing to do!
-		return nil, lastDir
-	}
-	if dir == lastDir {
-		// Still in same directory
-		return nil, lastDir
-	}
-	newLastDir = lastDir
-	for {
-		slashes := strings.Count(dir, "/")
-		if !strings.HasPrefix(lastDir, dir) && slashes < level {
-			dirNames = append([]string{dir}, dirNames...)
-		}
-		if newLastDir == lastDir {
-			newLastDir = dir
-		}
-		dir = path.Dir(dir)
-		if dir == "." {
-			break
-		}
-	}
-	return dirNames, newLastDir
-}
-
 // listFn is called from list to handle an object
 type listFn func(remote string, object *api.File, isDirectory bool) error
 
@@ -503,6 +475,8 @@ var errEndList = errors.New("end list")
 // list lists the objects into the function supplied from
 // the bucket and root supplied
 //
+// dir is the starting directory, "" for root
+//
 // level is the depth to search to
 //
 // If prefix is set then startFileName is used as a prefix which all
@@ -517,6 +491,14 @@ func (f *Fs) list(dir string, level int, prefix string, limit int, hidden bool,
 	if dir != "" {
 		root += dir + "/"
 	}
+	delimiter := ""
+	switch level {
+	case 1:
+		delimiter = "/"
+	case fs.MaxLevel:
+	default:
+		return fs.ErrorLevelNotSupported
+	}
 	bucketID, err := f.getBucketID()
 	if err != nil {
 		return err
@@ -528,6 +510,8 @@ func (f *Fs) list(dir string, level int, prefix string, limit int, hidden bool,
 	var request = api.ListFileNamesRequest{
 		BucketID:     bucketID,
 		MaxFileCount: chunkSize,
+		Prefix:       root,
+		Delimiter:    delimiter,
 	}
 	prefix = root + prefix
 	if prefix != "" {
@@ -541,7 +525,6 @@ func (f *Fs) list(dir string, level int, prefix string, limit int, hidden bool,
 	if hidden {
 		opts.Path = "/b2_list_file_versions"
 	}
-	lastDir := dir
 	for {
 		err := f.pacer.Call(func() (bool, error) {
 			resp, err := f.srv.CallJSON(&opts, &request, &response)
@@ -553,34 +536,26 @@ func (f *Fs) list(dir string, level int, prefix string, limit int, hidden bool,
 		for i := range response.Files {
 			file := &response.Files[i]
 			// Finish if file name no longer has prefix
-			if !strings.HasPrefix(file.Name, prefix) {
+			if prefix != "" && !strings.HasPrefix(file.Name, prefix) {
 				return nil
 			}
-			remote := file.Name[len(f.root):]
-			slashes := strings.Count(remote, "/")
-
-			// Check if this file makes a new directories
-			var dirNames []string
-			dirNames, lastDir = sendDir(lastDir, remote, level)
-			for _, dirName := range dirNames {
-				err = fn(dirName, nil, true)
-				if err != nil {
-					if err == errEndList {
-						return nil
-					}
-					return err
-				}
+			if !strings.HasPrefix(file.Name, f.root) {
+				fs.Log(f, "Odd name received %q", file.Name)
+				continue
 			}
-
-			// Send the file
-			if slashes < level {
-				err = fn(remote, file, false)
-				if err != nil {
-					if err == errEndList {
-						return nil
-					}
-					return err
+			remote := file.Name[len(f.root):]
+			// Check for directory
+			isDirectory := level != 0 && strings.HasSuffix(remote, "/")
+			if isDirectory {
+				remote = remote[:len(remote)-1]
+			}
+			// Send object
+			err = fn(remote, file, isDirectory)
+			if err != nil {
+				if err == errEndList {
+					return nil
 				}
+				return err
 			}
 		}
 		// end if no NextFileName

b2/b2_internal_test.go

@@ -5,7 +5,6 @@ import (
 	"time"
 
 	"github.com/ncw/rclone/fstest"
-	"github.com/stretchr/testify/assert"
 )
 
 // Test b2 string encoding
@@ -169,116 +168,3 @@ func TestParseTimeString(t *testing.T) {
 	}
 
 }
-
-func TestSendDir(t *testing.T) {
-	for _, test := range []struct {
-		lastDir    string
-		remote     string
-		level      int
-		dirNames   []string
-		newLastDir string
-	}{
-		{
-			lastDir:    "",
-			remote:     "test.txt",
-			level:      100,
-			dirNames:   nil,
-			newLastDir: "",
-		},
-		{
-			lastDir:    "",
-			remote:     "potato/test.txt",
-			level:      100,
-			dirNames:   []string{"potato"},
-			newLastDir: "potato",
-		},
-		{
-			lastDir:    "potato",
-			remote:     "potato/test.txt",
-			level:      100,
-			dirNames:   nil,
-			newLastDir: "potato",
-		},
-		{
-			lastDir:    "",
-			remote:     "potato/sausage/test.txt",
-			level:      100,
-			dirNames:   []string{"potato", "potato/sausage"},
-			newLastDir: "potato/sausage",
-		},
-		{
-			lastDir:    "potato",
-			remote:     "potato/sausage/test.txt",
-			level:      100,
-			dirNames:   []string{"potato/sausage"},
-			newLastDir: "potato/sausage",
-		},
-		{
-			lastDir:    "potato/sausage",
-			remote:     "potato/sausage/test.txt",
-			level:      100,
-			dirNames:   nil,
-			newLastDir: "potato/sausage",
-		},
-		{
-			lastDir:    "",
-			remote:     "a/b/c/d/e/f.txt",
-			level:      100,
-			dirNames:   []string{"a", "a/b", "a/b/c", "a/b/c/d", "a/b/c/d/e"},
-			newLastDir: "a/b/c/d/e",
-		},
-		{
-			lastDir:    "a/b/c/d/e",
-			remote:     "a/b/c/d/E/f.txt",
-			level:      100,
-			dirNames:   []string{"a/b/c/d/E"},
-			newLastDir: "a/b/c/d/E",
-		},
-		{
-			lastDir:    "a/b/c/d/e",
-			remote:     "a/b/C/D/E/f.txt",
-			level:      100,
-			dirNames:   []string{"a/b/C", "a/b/C/D", "a/b/C/D/E"},
-			newLastDir: "a/b/C/D/E",
-		},
-		{
-			lastDir:    "a/b/c",
-			remote:     "a/b/c/d/e/f.txt",
-			level:      100,
-			dirNames:   []string{"a/b/c/d", "a/b/c/d/e"},
-			newLastDir: "a/b/c/d/e",
-		},
-		{
-			lastDir:    "",
-			remote:     "a/b/c/d/e/f.txt",
-			level:      1,
-			dirNames:   []string{"a"},
-			newLastDir: "a/b/c/d/e",
-		},
-		{
-			lastDir:    "a/b/c",
-			remote:     "a/b/c/d/e/f.txt",
-			level:      1,
-			dirNames:   nil,
-			newLastDir: "a/b/c/d/e",
-		},
-		{
-			lastDir:    "",
-			remote:     "a/b/c/d/e/f.txt",
-			level:      3,
-			dirNames:   []string{"a", "a/b", "a/b/c"},
-			newLastDir: "a/b/c/d/e",
-		},
-		{
-			lastDir:    "a/b/C/D/E",
-			remote:     "a/b/c/d/e/f.txt",
-			level:      3,
-			dirNames:   []string{"a/b/c"},
-			newLastDir: "a/b/c/d/e",
-		},
-	} {
-		dirNames, newLastDir := sendDir(test.lastDir, test.remote, test.level)
-		assert.Equal(t, test.dirNames, dirNames, "dirNames fail for sendDir(%q,%q,%v)", test.lastDir, test.remote, test.level)
-		assert.Equal(t, test.newLastDir, newLastDir, "newLastDir fail for sendDir(%q,%q,%v)", test.lastDir, test.remote, test.level)
-	}
-}