From 50928a5027e208d202bebf663ddc70b5c4b36212 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 6 Jun 2017 16:40:00 +0100 Subject: [PATCH] Implement --fast-list flag. This is supported remotes which can do a recursive listing. It will use more memory. This is related to #1277 but doesn't fix that issue yet. --- docs/content/b2.md | 6 + docs/content/docs.md | 32 +++ docs/content/googlecloudstorage.md | 6 + docs/content/hubic.md | 6 + docs/content/overview.md | 36 ++-- docs/content/s3.md | 6 + docs/content/swift.md | 6 + docs/content/yandex.md | 6 + fs/config.go | 3 + fs/walk.go | 253 ++++++++++++++++++++++++ fs/walk_test.go | 302 ++++++++++++++++++++++++----- 11 files changed, 602 insertions(+), 60 deletions(-) diff --git a/docs/content/b2.md b/docs/content/b2.md index c095ba545..146a7456d 100644 --- a/docs/content/b2.md +++ b/docs/content/b2.md @@ -93,6 +93,12 @@ excess files in the bucket. rclone sync /home/local/directory remote:bucket +### --fast-list ### + +This remote supports `--fast-list` which allows you to use fewer +transactions in exchange for more memory. See the [rclone +docs](/docs/#fast-list) for more details. + ### Modified time ### The modified time is stored as metadata on the object as diff --git a/docs/content/docs.md b/docs/content/docs.md index 35f8afbc9..6c55a323a 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -581,6 +581,38 @@ errors subsequent to that. If there have been errors before the deletions start then you will get the message `not deleting files as there were IO errors`. +### --fast-list ### + +When doing anything which involves a directory listing (eg `sync`, +`copy`, `ls` - in fact nearly every command), rclone normally lists a +directory and processes it before using more directory lists to +process any subdirectories. This can be parallelised and works very +quickly using the least amount of memory. + +However some remotes have a way of listing all files beneath a +directory in one (or a small number) of transactions. These tend to +be the bucket based remotes (eg s3, b2, gcs, swift, hubic). + +If you use the `--fast-list` flag then rclone will use this method for +listing directories. This will have the following consequences for +the listing: + + * It **will** use fewer transactions (important if you pay for them) + * It **will** use more memory. Rclone has to load the whole listing into memory. + * It *may* be faster because it uses fewer transactions + * It *may* be slower because it can't be parallelized + +rclone should always give identical results with and without +`--fast-list`. + +If you pay for transactions and can fit your entire sync listing into +memory then `--fast-list` is recommended. If you have a very big sync +to do then don't use `--fast-list` otherwise you will run out of +memory. + +If you use `--fast-list` on a remote which doesn't support it, then +rclone will just ignore it. + ### --timeout=TIME ### This sets the IO idle timeout. If a transfer has started but then diff --git a/docs/content/googlecloudstorage.md b/docs/content/googlecloudstorage.md index 034315b29..77ffda552 100644 --- a/docs/content/googlecloudstorage.md +++ b/docs/content/googlecloudstorage.md @@ -168,6 +168,12 @@ to your Service Account credentials at the `service_account_file` prompt and rclone won't use the browser based authentication flow. +### --fast-list ### + +This remote supports `--fast-list` which allows you to use fewer +transactions in exchange for more memory. See the [rclone +docs](/docs/#fast-list) for more details. + ### Modified time ### Google google cloud storage stores md5sums natively and rclone stores diff --git a/docs/content/hubic.md b/docs/content/hubic.md index 311bb7da6..1156b7c49 100644 --- a/docs/content/hubic.md +++ b/docs/content/hubic.md @@ -110,6 +110,12 @@ browser*, you need to copy your files to the `default` directory rclone copy /home/source remote:default/backup +### --fast-list ### + +This remote supports `--fast-list` which allows you to use fewer +transactions in exchange for more memory. See the [rclone +docs](/docs/#fast-list) for more details. + ### Modified time ### The modified time is stored as metadata on the object as diff --git a/docs/content/overview.md b/docs/content/overview.md index f88d1415d..971c991c7 100644 --- a/docs/content/overview.md +++ b/docs/content/overview.md @@ -108,21 +108,21 @@ All the remotes support a basic set of features, but there are some optional features supported by some remotes used to make some operations more efficient. -| Name | Purge | Copy | Move | DirMove | CleanUp | -| ---------------------- |:-----:|:----:|:----:|:-------:|:-------:| -| Google Drive | Yes | Yes | Yes | Yes | No [#575](https://github.com/ncw/rclone/issues/575) | -| Amazon S3 | No | Yes | No | No | No | -| Openstack Swift | Yes † | Yes | No | No | No | -| Dropbox | Yes | Yes | Yes | Yes | No [#575](https://github.com/ncw/rclone/issues/575) | -| Google Cloud Storage | Yes | Yes | No | No | No | -| Amazon Drive | Yes | No | Yes | Yes | No [#575](https://github.com/ncw/rclone/issues/575) | -| Microsoft OneDrive | Yes | Yes | Yes | No [#197](https://github.com/ncw/rclone/issues/197) | No [#575](https://github.com/ncw/rclone/issues/575) | -| Hubic | Yes † | Yes | No | No | No | -| Backblaze B2 | No | No | No | No | Yes | -| Yandex Disk | Yes | No | No | No | No [#575](https://github.com/ncw/rclone/issues/575) | -| SFTP | No | No | Yes | Yes | No | -| FTP | No | No | Yes | Yes | No | -| The local filesystem | Yes | No | Yes | Yes | No | +| Name | Purge | Copy | Move | DirMove | CleanUp | ListR | +| ---------------------- |:-----:|:----:|:----:|:-------:|:-------:|:-----:| +| Google Drive | Yes | Yes | Yes | Yes | No [#575](https://github.com/ncw/rclone/issues/575) | No | +| Amazon S3 | No | Yes | No | No | No | Yes | +| Openstack Swift | Yes † | Yes | No | No | No | Yes | +| Dropbox | Yes | Yes | Yes | Yes | No [#575](https://github.com/ncw/rclone/issues/575) | No | +| Google Cloud Storage | Yes | Yes | No | No | No | Yes | +| Amazon Drive | Yes | No | Yes | Yes | No [#575](https://github.com/ncw/rclone/issues/575) | No | +| Microsoft OneDrive | Yes | Yes | Yes | No [#197](https://github.com/ncw/rclone/issues/197) | No [#575](https://github.com/ncw/rclone/issues/575) | No | +| Hubic | Yes † | Yes | No | No | No | Yes | +| Backblaze B2 | No | No | No | No | Yes | Yes | +| Yandex Disk | Yes | No | No | No | No [#575](https://github.com/ncw/rclone/issues/575) | Yes | +| SFTP | No | No | Yes | Yes | No | No | +| FTP | No | No | Yes | Yes | No | No | +| The local filesystem | Yes | No | Yes | Yes | No | No | ### Purge ### @@ -166,3 +166,9 @@ This is used for emptying the trash for a remote by `rclone cleanup`. If the server can't do `CleanUp` then `rclone cleanup` will return an error. + +### ListR ### + +The remote supports a recursive list to list all the contents beneath +a directory quickly. This enables the `--fast-list` flag to work. +See the [rclone docs](/docs/#fast-list) for more details. diff --git a/docs/content/s3.md b/docs/content/s3.md index 4ee8648bc..5e600562d 100644 --- a/docs/content/s3.md +++ b/docs/content/s3.md @@ -210,6 +210,12 @@ files in the bucket. rclone sync /home/local/directory remote:bucket +### --fast-list ### + +This remote supports `--fast-list` which allows you to use fewer +transactions in exchange for more memory. See the [rclone +docs](/docs/#fast-list) for more details. + ### Modified time ### The modified time is stored as metadata on the object as diff --git a/docs/content/swift.md b/docs/content/swift.md index 7686f7013..e95572c93 100644 --- a/docs/content/swift.md +++ b/docs/content/swift.md @@ -158,6 +158,12 @@ tenant = $OS_TENANT_NAME Note that you may (or may not) need to set `region` too - try without first. +### --fast-list ### + +This remote supports `--fast-list` which allows you to use fewer +transactions in exchange for more memory. See the [rclone +docs](/docs/#fast-list) for more details. + ### Specific options ### Here are the command line options specific to this cloud storage diff --git a/docs/content/yandex.md b/docs/content/yandex.md index 0c7eebdb5..77e736986 100644 --- a/docs/content/yandex.md +++ b/docs/content/yandex.md @@ -107,6 +107,12 @@ excess files in the path. rclone sync /home/local/directory remote:directory +### --fast-list ### + +This remote supports `--fast-list` which allows you to use fewer +transactions in exchange for more memory. See the [rclone +docs](/docs/#fast-list) for more details. + ### Modified time ### Modified times are supported and are stored accurate to 1 ns in custom diff --git a/fs/config.go b/fs/config.go index 940d766b6..7573adbc1 100644 --- a/fs/config.go +++ b/fs/config.go @@ -96,6 +96,7 @@ var ( noUpdateModTime = BoolP("no-update-modtime", "", false, "Don't update destination mod-time if files identical.") backupDir = StringP("backup-dir", "", "", "Make backups into hierarchy based in DIR.") suffix = StringP("suffix", "", "", "Suffix for use with --backup-dir.") + useListR = BoolP("fast-list", "", false, "Use recursive list if available. Uses more memory but fewer transactions.") bwLimit BwTimetable bufferSize SizeSuffix = 16 << 20 @@ -221,6 +222,7 @@ type ConfigInfo struct { DataRateUnit string BackupDir string Suffix string + UseListR bool BufferSize SizeSuffix } @@ -367,6 +369,7 @@ func LoadConfig() { Config.NoUpdateModTime = *noUpdateModTime Config.BackupDir = *backupDir Config.Suffix = *suffix + Config.UseListR = *useListR Config.BufferSize = bufferSize ConfigPath = *configFile diff --git a/fs/walk.go b/fs/walk.go index 1281c9e34..ac6b97743 100644 --- a/fs/walk.go +++ b/fs/walk.go @@ -3,6 +3,11 @@ package fs import ( + "bytes" + "fmt" + "path" + "sort" + "strings" "sync" "github.com/pkg/errors" @@ -13,6 +18,10 @@ import ( // an error by any function. var ErrorSkipDir = errors.New("skip this directory") +// ErrorCantListR is returned by WalkR if the underlying Fs isn't +// capable of doing a recursive listing. +var ErrorCantListR = errors.New("recursive directory listing not available") + // WalkFunc is the type of the function called for directory // visited by Walk. The path argument contains remote path to the directory. // @@ -39,11 +48,32 @@ type WalkFunc func(path string, entries DirEntries, err error) error // // Parent directories are always listed before their children // +// This is implemented by WalkR if Config.UseRecursiveListing is true +// and f supports it and level > 1, or WalkN otherwise. +// // NB (f, path) to be replaced by fs.Dir at some point func Walk(f Fs, path string, includeAll bool, maxLevel int, fn WalkFunc) error { + if (maxLevel < 0 || maxLevel > 1) && Config.UseListR && f.Features().ListR != nil { + return WalkR(f, path, includeAll, maxLevel, fn) + } + return WalkN(f, path, includeAll, maxLevel, fn) +} + +// WalkN lists the directory. +// +// It implements Walk using non recursive directory listing. +func WalkN(f Fs, path string, includeAll bool, maxLevel int, fn WalkFunc) error { return walk(f, path, includeAll, maxLevel, fn, ListDirSorted) } +// WalkR lists the directory. +// +// It implements Walk using recursive directory listing if +// available, or returns ErrorCantListR if not. +func WalkR(f Fs, path string, includeAll bool, maxLevel int, fn WalkFunc) error { + return walkR(f, path, includeAll, maxLevel, fn, listR) +} + type listDirFunc func(fs Fs, includeAll bool, dir string) (entries DirEntries, err error) func walk(f Fs, path string, includeAll bool, maxLevel int, fn WalkFunc, listDir listDirFunc) error { @@ -139,6 +169,229 @@ func walk(f Fs, path string, includeAll bool, maxLevel int, fn WalkFunc, listDir return <-errs } +// DirTree is a map of directories to entries +type DirTree map[string]DirEntries + +// parentDir finds the parent directory of path +func parentDir(entryPath string) string { + dirPath := path.Dir(entryPath) + if dirPath == "." { + dirPath = "" + } + return dirPath +} + +// add an entry to the tree +func (dt DirTree) add(entry BasicInfo) { + dirPath := parentDir(entry.Remote()) + dt[dirPath] = append(dt[dirPath], entry) +} + +// add a directory entry to the tree +func (dt DirTree) addDir(entry BasicInfo) { + dt.add(entry) + // create the directory itself if it doesn't exist already + dirPath := entry.Remote() + if _, ok := dt[dirPath]; !ok { + dt[dirPath] = nil + } +} + +// check that dirPath has a *Dir in its parent +func (dt DirTree) checkParent(root, dirPath string) { + if dirPath == root { + return + } + parentPath := parentDir(dirPath) + entries := dt[parentPath] + for _, entry := range entries { + if entry.Remote() == dirPath { + return + } + } + dt[parentPath] = append(entries, &Dir{ + Name: dirPath, + }) + dt.checkParent(root, parentPath) +} + +// check every directory in the tree has *Dir in its parent +func (dt DirTree) checkParents(root string) { + for dirPath := range dt { + dt.checkParent(root, dirPath) + } +} + +// Sort sorts all the Entries +func (dt DirTree) Sort() { + for _, entries := range dt { + sort.Sort(entries) + } +} + +// Dirs returns the directories in sorted order +func (dt DirTree) Dirs() (dirNames []string) { + for dirPath := range dt { + dirNames = append(dirNames, dirPath) + } + sort.Strings(dirNames) + return dirNames +} + +// String emits a simple representation of the DirTree +func (dt DirTree) String() string { + out := new(bytes.Buffer) + for _, dir := range dt.Dirs() { + fmt.Fprintf(out, "%s/\n", dir) + for _, entry := range dt[dir] { + flag := "" + if _, ok := entry.(*Dir); ok { + flag = "/" + } + fmt.Fprintf(out, " %s%s\n", path.Base(entry.Remote()), flag) + } + } + return out.String() +} + +type listRCallback func(entries DirEntries) error + +type listRFunc func(f Fs, dir string, callback listRCallback) error + +// FIXME Pretend ListR function +func listR(f Fs, dir string, callback listRCallback) (err error) { + listR := f.Features().ListR + if listR == nil { + return ErrorCantListR + } + const maxEntries = 100 + entries := make(DirEntries, 0, maxEntries) + list := NewLister() + list.Start(f, dir) + for { + o, dir, err := list.Get() + if err != nil { + return err + } else if o != nil { + entries = append(entries, o) + } else if dir != nil { + entries = append(entries, dir) + } else { + // finishd since err, o, dir == nil + break + } + if len(entries) >= maxEntries { + err = callback(entries) + if err != nil { + return err + } + entries = entries[:0] + } + } + err = list.Error() + if err != nil { + return err + } + if len(entries) > 0 { + err = callback(entries) + if err != nil { + return err + } + } + return nil + +} + +func walkRDirTree(f Fs, path string, includeAll bool, maxLevel int, listRFn listRFunc) (DirTree, error) { + dirs := make(DirTree) + err := listRFn(f, path, func(entries DirEntries) error { + for _, entry := range entries { + slashes := strings.Count(entry.Remote(), "/") + switch x := entry.(type) { + case Object: + // Make sure we don't delete excluded files if not required + if includeAll || Config.Filter.IncludeObject(x) { + if maxLevel < 0 || slashes <= maxLevel-1 { + dirs.add(x) + } else { + // Make sure we include any parent directories of excluded objects + dirPath := x.Remote() + for ; slashes > maxLevel-1; slashes-- { + dirPath = parentDir(dirPath) + } + dirs.checkParent(path, dirPath) + } + } else { + Debugf(x, "Excluded from sync (and deletion)") + } + case *Dir: + if includeAll || Config.Filter.IncludeDirectory(x.Remote()) { + if maxLevel < 0 || slashes <= maxLevel-1 { + if slashes == maxLevel-1 { + // Just add the object if at maxLevel + dirs.add(x) + } else { + dirs.addDir(x) + } + } + } else { + Debugf(x, "Excluded from sync (and deletion)") + } + } + } + return nil + }) + if err != nil { + return nil, err + } + dirs.checkParents(path) + if len(dirs) == 0 { + dirs[path] = nil + } + dirs.Sort() + return dirs, nil +} + +// NewDirTree returns a DirTree filled with the directory listing using the parameters supplied +func NewDirTree(f Fs, path string, includeAll bool, maxLevel int) (DirTree, error) { + return walkRDirTree(f, path, includeAll, maxLevel, listR) +} + +func walkR(f Fs, path string, includeAll bool, maxLevel int, fn WalkFunc, listRFn listRFunc) error { + dirs, err := walkRDirTree(f, path, includeAll, maxLevel, listRFn) + if err != nil { + return err + } + skipping := false + skipPrefix := "" + emptyDir := DirEntries{} + for _, dirPath := range dirs.Dirs() { + if skipping { + // Skip over directories as required + if strings.HasPrefix(dirPath, skipPrefix) { + continue + } + skipping = false + } + entries := dirs[dirPath] + if entries == nil { + entries = emptyDir + } + sort.Sort(entries) + err = fn(dirPath, entries, nil) + if err == ErrorSkipDir { + skipping = true + skipPrefix = dirPath + if skipPrefix != "" { + skipPrefix += "/" + } + } else if err != nil { + return err + } + } + return nil +} + // WalkGetAll runs Walk getting all the results func WalkGetAll(f Fs, path string, includeAll bool, maxLevel int) (objs []Object, dirs []*Dir, err error) { err = Walk(f, path, includeAll, maxLevel, func(dirPath string, entries DirEntries, err error) error { diff --git a/fs/walk_test.go b/fs/walk_test.go index c61c09e6c..1ee2461b9 100644 --- a/fs/walk_test.go +++ b/fs/walk_test.go @@ -1,11 +1,13 @@ package fs import ( + "fmt" "sync" "testing" "github.com/pkg/errors" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) type ( @@ -79,6 +81,30 @@ func (ls *listDirs) ListDir(f Fs, includeAll bool, dir string) (entries DirEntri return result.entries, result.err } +// ListR returns the expected listing for the directory using ListR +func (ls *listDirs) ListR(f Fs, dir string, callback listRCallback) (err error) { + ls.mu.Lock() + defer ls.mu.Unlock() + assert.Equal(ls.t, ls.fs, f) + //assert.Equal(ls.t, ls.includeAll, includeAll) + + var errorReturn error + for dirPath, result := range ls.results { + // Put expected results for call of WalkFn + // Note that we don't call the function at all if we got an error + if result.err != nil { + errorReturn = result.err + } + if errorReturn == nil { + err = callback(result.entries) + require.NoError(ls.t, err) + ls.walkResults[dirPath] = result + } + } + ls.results = listResults{} + return errorReturn +} + // IsFinished checks everything expected was used up func (ls *listDirs) IsFinished() { if ls.checkMaps { @@ -92,6 +118,7 @@ func (ls *listDirs) IsFinished() { func (ls *listDirs) WalkFn(dir string, entries DirEntries, err error) error { ls.mu.Lock() defer ls.mu.Unlock() + // ls.t.Logf("WalkFn(%q, %v, %q)", dir, entries, err) // Fetch expected entries and err result, ok := ls.walkResults[dir] @@ -123,12 +150,21 @@ func (ls *listDirs) Walk() { ls.IsFinished() } +// WalkR does the walkR and tests the expectations +func (ls *listDirs) WalkR() { + err := walkR(nil, "", ls.includeAll, ls.maxLevel, ls.WalkFn, ls.ListR) + assert.Equal(ls.t, ls.finalError, err) + if ls.finalError == nil { + ls.IsFinished() + } +} + func newDir(name string) *Dir { return &Dir{Name: name} } -func TestWalkEmpty(t *testing.T) { - newListDirs(t, nil, false, +func testWalkEmpty(t *testing.T) *listDirs { + return newListDirs(t, nil, false, listResults{ "": {entries: DirEntries{}, err: nil}, }, @@ -136,11 +172,13 @@ func TestWalkEmpty(t *testing.T) { "": nil, }, nil, - ).Walk() + ) } +func TestWalkEmpty(t *testing.T) { testWalkEmpty(t).Walk() } +func TestWalkREmpty(t *testing.T) { testWalkEmpty(t).WalkR() } -func TestWalkEmptySkip(t *testing.T) { - newListDirs(t, nil, true, +func testWalkEmptySkip(t *testing.T) *listDirs { + return newListDirs(t, nil, true, listResults{ "": {entries: DirEntries{}, err: nil}, }, @@ -148,11 +186,13 @@ func TestWalkEmptySkip(t *testing.T) { "": ErrorSkipDir, }, nil, - ).Walk() + ) } +func TestWalkEmptySkip(t *testing.T) { testWalkEmptySkip(t).Walk() } +func TestWalkREmptySkip(t *testing.T) { testWalkEmptySkip(t).WalkR() } -func TestWalkNotFound(t *testing.T) { - newListDirs(t, nil, true, +func testWalkNotFound(t *testing.T) *listDirs { + return newListDirs(t, nil, true, listResults{ "": {err: ErrorDirNotFound}, }, @@ -160,10 +200,13 @@ func TestWalkNotFound(t *testing.T) { "": ErrorDirNotFound, }, ErrorDirNotFound, - ).Walk() + ) } +func TestWalkNotFound(t *testing.T) { testWalkNotFound(t).Walk() } +func TestWalkRNotFound(t *testing.T) { testWalkNotFound(t).WalkR() } func TestWalkNotFoundMaskError(t *testing.T) { + // this doesn't work for WalkR newListDirs(t, nil, true, listResults{ "": {err: ErrorDirNotFound}, @@ -176,6 +219,7 @@ func TestWalkNotFoundMaskError(t *testing.T) { } func TestWalkNotFoundSkipkError(t *testing.T) { + // this doesn't work for WalkR newListDirs(t, nil, true, listResults{ "": {err: ErrorDirNotFound}, @@ -187,17 +231,21 @@ func TestWalkNotFoundSkipkError(t *testing.T) { ).Walk() } -func testWalkLevels(t *testing.T, maxLevel int) { +func testWalkLevels(t *testing.T, maxLevel int) *listDirs { da := newDir("a") + oA := mockObject("A") db := newDir("a/b") + oB := mockObject("a/B") dc := newDir("a/b/c") + oC := mockObject("a/b/C") dd := newDir("a/b/c/d") - newListDirs(t, nil, false, + oD := mockObject("a/b/c/D") + return newListDirs(t, nil, false, listResults{ - "": {entries: DirEntries{da}, err: nil}, - "a": {entries: DirEntries{db}, err: nil}, - "a/b": {entries: DirEntries{dc}, err: nil}, - "a/b/c": {entries: DirEntries{dd}, err: nil}, + "": {entries: DirEntries{oA, da}, err: nil}, + "a": {entries: DirEntries{oB, db}, err: nil}, + "a/b": {entries: DirEntries{oC, dc}, err: nil}, + "a/b/c": {entries: DirEntries{oD, dd}, err: nil}, "a/b/c/d": {entries: DirEntries{}, err: nil}, }, errorMap{ @@ -208,51 +256,54 @@ func testWalkLevels(t *testing.T, maxLevel int) { "a/b/c/d": nil, }, nil, - ).SetLevel(maxLevel).Walk() + ).SetLevel(maxLevel) } +func TestWalkLevels(t *testing.T) { testWalkLevels(t, -1).Walk() } +func TestWalkRLevels(t *testing.T) { testWalkLevels(t, -1).WalkR() } +func TestWalkLevelsNoRecursive10(t *testing.T) { testWalkLevels(t, 10).Walk() } +func TestWalkRLevelsNoRecursive10(t *testing.T) { testWalkLevels(t, 10).WalkR() } -func TestWalkLevels(t *testing.T) { - testWalkLevels(t, -1) -} - -func TestWalkLevelsNoRecursive10(t *testing.T) { - testWalkLevels(t, 10) -} - -func TestWalkLevelsNoRecursive(t *testing.T) { +func testWalkLevelsNoRecursive(t *testing.T) *listDirs { da := newDir("a") - newListDirs(t, nil, false, + oA := mockObject("A") + return newListDirs(t, nil, false, listResults{ - "": {entries: DirEntries{da}, err: nil}, + "": {entries: DirEntries{oA, da}, err: nil}, }, errorMap{ "": nil, }, nil, - ).SetLevel(1).Walk() + ).SetLevel(1) } +func TestWalkLevelsNoRecursive(t *testing.T) { testWalkLevelsNoRecursive(t).Walk() } +func TestWalkRLevelsNoRecursive(t *testing.T) { testWalkLevelsNoRecursive(t).WalkR() } -func TestWalkLevels2(t *testing.T) { +func testWalkLevels2(t *testing.T) *listDirs { da := newDir("a") + oA := mockObject("A") db := newDir("a/b") - newListDirs(t, nil, false, + oB := mockObject("a/B") + return newListDirs(t, nil, false, listResults{ - "": {entries: DirEntries{da}, err: nil}, - "a": {entries: DirEntries{db}, err: nil}, + "": {entries: DirEntries{oA, da}, err: nil}, + "a": {entries: DirEntries{oB, db}, err: nil}, }, errorMap{ "": nil, "a": nil, }, nil, - ).SetLevel(2).Walk() + ).SetLevel(2) } +func TestWalkLevels2(t *testing.T) { testWalkLevels2(t).Walk() } +func TestWalkRLevels2(t *testing.T) { testWalkLevels2(t).WalkR() } -func TestWalkSkip(t *testing.T) { +func testWalkSkip(t *testing.T) *listDirs { da := newDir("a") db := newDir("a/b") dc := newDir("a/b/c") - newListDirs(t, nil, false, + return newListDirs(t, nil, false, listResults{ "": {entries: DirEntries{da}, err: nil}, "a": {entries: DirEntries{db}, err: nil}, @@ -264,10 +315,12 @@ func TestWalkSkip(t *testing.T) { "a/b": ErrorSkipDir, }, nil, - ).Walk() + ) } +func TestWalkSkip(t *testing.T) { testWalkSkip(t).Walk() } +func TestWalkRSkip(t *testing.T) { testWalkSkip(t).WalkR() } -func TestWalkErrors(t *testing.T) { +func testWalkErrors(t *testing.T) *listDirs { lr := listResults{} em := errorMap{} de := make(DirEntries, 10) @@ -279,12 +332,14 @@ func TestWalkErrors(t *testing.T) { } lr[""] = listResult{entries: de, err: nil} em[""] = nil - newListDirs(t, nil, true, + return newListDirs(t, nil, true, lr, em, ErrorDirNotFound, - ).NoCheckMaps().Walk() + ).NoCheckMaps() } +func TestWalkErrors(t *testing.T) { testWalkErrors(t).Walk() } +func TestWalkRErrors(t *testing.T) { testWalkErrors(t).WalkR() } var errorBoom = errors.New("boom") @@ -314,20 +369,177 @@ func makeTree(level int, terminalErrors bool) (listResults, errorMap) { return lr, em } -func TestWalkMulti(t *testing.T) { +func testWalkMulti(t *testing.T) *listDirs { lr, em := makeTree(3, false) - newListDirs(t, nil, true, + return newListDirs(t, nil, true, lr, em, nil, - ).Walk() + ) } +func TestWalkMulti(t *testing.T) { testWalkMulti(t).Walk() } +func TestWalkRMulti(t *testing.T) { testWalkMulti(t).WalkR() } -func TestWalkMultiErrors(t *testing.T) { +func testWalkMultiErrors(t *testing.T) *listDirs { lr, em := makeTree(3, true) - newListDirs(t, nil, true, + return newListDirs(t, nil, true, lr, em, errorBoom, - ).NoCheckMaps().Walk() + ).NoCheckMaps() +} +func TestWalkMultiErrors(t *testing.T) { testWalkMultiErrors(t).Walk() } +func TestWalkRMultiErrors(t *testing.T) { testWalkMultiErrors(t).Walk() } + +// a very simple listRcallback function +func makeListRCallback(entries DirEntries, err error) listRFunc { + return func(f Fs, dir string, callback listRCallback) error { + if err == nil { + err = callback(entries) + } + return err + } +} + +func TestWalkRDirTree(t *testing.T) { + for _, test := range []struct { + entries DirEntries + want string + err error + root string + level int + }{ + {DirEntries{}, "/\n", nil, "", -1}, + {DirEntries{mockObject("a")}, `/ + a +`, nil, "", -1}, + {DirEntries{mockObject("a/b")}, `/ + a/ +a/ + b +`, nil, "", -1}, + {DirEntries{mockObject("a/b/c/d")}, `/ + a/ +a/ + b/ +a/b/ + c/ +a/b/c/ + d +`, nil, "", -1}, + {DirEntries{mockObject("a")}, "", errorBoom, "", -1}, + {DirEntries{ + mockObject("0/1/2/3"), + mockObject("4/5/6/7"), + mockObject("8/9/a/b"), + mockObject("c/d/e/f"), + mockObject("g/h/i/j"), + mockObject("k/l/m/n"), + mockObject("o/p/q/r"), + mockObject("s/t/u/v"), + mockObject("w/x/y/z"), + }, `/ + 0/ + 4/ + 8/ + c/ + g/ + k/ + o/ + s/ + w/ +0/ + 1/ +0/1/ + 2/ +0/1/2/ + 3 +4/ + 5/ +4/5/ + 6/ +4/5/6/ + 7 +8/ + 9/ +8/9/ + a/ +8/9/a/ + b +c/ + d/ +c/d/ + e/ +c/d/e/ + f +g/ + h/ +g/h/ + i/ +g/h/i/ + j +k/ + l/ +k/l/ + m/ +k/l/m/ + n +o/ + p/ +o/p/ + q/ +o/p/q/ + r +s/ + t/ +s/t/ + u/ +s/t/u/ + v +w/ + x/ +w/x/ + y/ +w/x/y/ + z +`, nil, "", -1}, + {DirEntries{ + mockObject("a/b/c/d/e/f1"), + mockObject("a/b/c/d/e/f2"), + mockObject("a/b/c/d/e/f3"), + }, `a/b/c/ + d/ +a/b/c/d/ + e/ +a/b/c/d/e/ + f1 + f2 + f3 +`, nil, "a/b/c", -1}, + {DirEntries{ + mockObject("A"), + mockObject("a/B"), + mockObject("a/b/C"), + mockObject("a/b/c/D"), + mockObject("a/b/c/d/E"), + }, `/ + A + a/ +a/ + B + b/ +`, nil, "", 2}, + {DirEntries{ + mockObject("a/b/c"), + mockObject("a/b/c/d/e"), + }, `/ + a/ +a/ + b/ +`, nil, "", 2}, + } { + r, err := walkRDirTree(nil, test.root, true, test.level, makeListRCallback(test.entries, test.err)) + assert.Equal(t, test.err, err, fmt.Sprintf("%+v", test)) + assert.Equal(t, test.want, r.String(), fmt.Sprintf("%+v", test)) + } }