sync,copy: Fix --fast-list --create-empty-src-dirs and --exclude
Before this change, if --fast-list was in use while doing a sync or copy with --create-empty-src-dirs and --exclude excluded all the files from the directory (but not the directory), then the directory would not be created. This is also visible with `rclone tree` which uses the same tree building approach as `rclone sync --fast-list` where the directories would go missing from the tree view. This was caused by not adding the parents of excluded files to the directory tree. See: https://forum.rclone.org/t/create-empty-src-dirs-issue-with-b2/30856
This commit is contained in:
Nick Craig-Wood
parent
28e43fe7af
commit
da404dc0f2
|
|
@ -471,23 +471,39 @@ func walkRDirTree(ctx context.Context, f fs.Fs, startPath string, includeAll boo
|
|||
defer mu.Unlock()
|
||||
for _, entry := range entries {
|
||||
slashes := strings.Count(entry.Remote(), "/")
|
||||
excluded := true
|
||||
switch x := entry.(type) {
|
||||
case fs.Object:
|
||||
// Make sure we don't delete excluded files if not required
|
||||
if includeAll || fi.IncludeObject(ctx, x) {
|
||||
if maxLevel < 0 || slashes <= maxLevel-1 {
|
||||
dirs.Add(x)
|
||||
} else {
|
||||
// Make sure we include any parent directories of excluded objects
|
||||
dirPath := x.Remote()
|
||||
for ; slashes > maxLevel-1; slashes-- {
|
||||
dirPath = parentDir(dirPath)
|
||||
}
|
||||
dirs.CheckParent(startPath, dirPath)
|
||||
excluded = false
|
||||
}
|
||||
} else {
|
||||
fs.Debugf(x, "Excluded from sync (and deletion)")
|
||||
}
|
||||
// Make sure we include any parent directories of excluded objects
|
||||
if excluded {
|
||||
dirPath := parentDir(x.Remote())
|
||||
slashes--
|
||||
if maxLevel >= 0 {
|
||||
for ; slashes > maxLevel-1; slashes-- {
|
||||
dirPath = parentDir(dirPath)
|
||||
}
|
||||
}
|
||||
inc, err := includeDirectory(dirPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if inc || includeAll {
|
||||
// If the directory doesn't exist already, create it
|
||||
_, obj := dirs.Find(dirPath)
|
||||
if obj == nil {
|
||||
dirs.AddDir(fs.NewDir(dirPath, time.Now()))
|
||||
}
|
||||
}
|
||||
}
|
||||
// Check if we need to prune a directory later.
|
||||
if !includeAll && len(fi.Opt.ExcludeFile) > 0 {
|
||||
basename := path.Base(x.Remote())
|
||||
|
|
|
|||
|
|
@ -448,17 +448,32 @@ func TestWalkRDirTree(t *testing.T) {
|
|||
err error
|
||||
root string
|
||||
level int
|
||||
exclude string
|
||||
}{
|
||||
{fs.DirEntries{}, "/\n", nil, "", -1},
|
||||
{fs.DirEntries{mockobject.Object("a")}, `/
|
||||
{
|
||||
entries: fs.DirEntries{},
|
||||
want: "/\n",
|
||||
level: -1,
|
||||
},
|
||||
{
|
||||
entries: fs.DirEntries{mockobject.Object("a")},
|
||||
want: `/
|
||||
a
|
||||
`, nil, "", -1},
|
||||
{fs.DirEntries{mockobject.Object("a/b")}, `/
|
||||
`,
|
||||
level: -1,
|
||||
},
|
||||
{
|
||||
entries: fs.DirEntries{mockobject.Object("a/b")},
|
||||
want: `/
|
||||
a/
|
||||
a/
|
||||
b
|
||||
`, nil, "", -1},
|
||||
{fs.DirEntries{mockobject.Object("a/b/c/d")}, `/
|
||||
`,
|
||||
level: -1,
|
||||
},
|
||||
{
|
||||
entries: fs.DirEntries{mockobject.Object("a/b/c/d")},
|
||||
want: `/
|
||||
a/
|
||||
a/
|
||||
b/
|
||||
|
|
@ -466,19 +481,27 @@ a/b/
|
|||
c/
|
||||
a/b/c/
|
||||
d
|
||||
`, nil, "", -1},
|
||||
{fs.DirEntries{mockobject.Object("a")}, "", errorBoom, "", -1},
|
||||
{fs.DirEntries{
|
||||
mockobject.Object("0/1/2/3"),
|
||||
mockobject.Object("4/5/6/7"),
|
||||
mockobject.Object("8/9/a/b"),
|
||||
mockobject.Object("c/d/e/f"),
|
||||
mockobject.Object("g/h/i/j"),
|
||||
mockobject.Object("k/l/m/n"),
|
||||
mockobject.Object("o/p/q/r"),
|
||||
mockobject.Object("s/t/u/v"),
|
||||
mockobject.Object("w/x/y/z"),
|
||||
}, `/
|
||||
`,
|
||||
level: -1,
|
||||
},
|
||||
{
|
||||
entries: fs.DirEntries{mockobject.Object("a")},
|
||||
err: errorBoom,
|
||||
level: -1,
|
||||
},
|
||||
{
|
||||
entries: fs.DirEntries{
|
||||
mockobject.Object("0/1/2/3"),
|
||||
mockobject.Object("4/5/6/7"),
|
||||
mockobject.Object("8/9/a/b"),
|
||||
mockobject.Object("c/d/e/f"),
|
||||
mockobject.Object("g/h/i/j"),
|
||||
mockobject.Object("k/l/m/n"),
|
||||
mockobject.Object("o/p/q/r"),
|
||||
mockobject.Object("s/t/u/v"),
|
||||
mockobject.Object("w/x/y/z"),
|
||||
},
|
||||
want: `/
|
||||
0/
|
||||
4/
|
||||
8/
|
||||
|
|
@ -542,12 +565,16 @@ w/x/
|
|||
y/
|
||||
w/x/y/
|
||||
z
|
||||
`, nil, "", -1},
|
||||
{fs.DirEntries{
|
||||
mockobject.Object("a/b/c/d/e/f1"),
|
||||
mockobject.Object("a/b/c/d/e/f2"),
|
||||
mockobject.Object("a/b/c/d/e/f3"),
|
||||
}, `a/b/c/
|
||||
`,
|
||||
level: -1,
|
||||
},
|
||||
{
|
||||
entries: fs.DirEntries{
|
||||
mockobject.Object("a/b/c/d/e/f1"),
|
||||
mockobject.Object("a/b/c/d/e/f2"),
|
||||
mockobject.Object("a/b/c/d/e/f3"),
|
||||
},
|
||||
want: `a/b/c/
|
||||
d/
|
||||
a/b/c/d/
|
||||
e/
|
||||
|
|
@ -555,32 +582,91 @@ a/b/c/d/e/
|
|||
f1
|
||||
f2
|
||||
f3
|
||||
`, nil, "a/b/c", -1},
|
||||
{fs.DirEntries{
|
||||
mockobject.Object("A"),
|
||||
mockobject.Object("a/B"),
|
||||
mockobject.Object("a/b/C"),
|
||||
mockobject.Object("a/b/c/D"),
|
||||
mockobject.Object("a/b/c/d/E"),
|
||||
}, `/
|
||||
`,
|
||||
root: "a/b/c",
|
||||
level: -1,
|
||||
},
|
||||
{
|
||||
entries: fs.DirEntries{
|
||||
mockobject.Object("A"),
|
||||
mockobject.Object("a/B"),
|
||||
mockobject.Object("a/b/C"),
|
||||
mockobject.Object("a/b/c/D"),
|
||||
mockobject.Object("a/b/c/d/E"),
|
||||
},
|
||||
want: `/
|
||||
A
|
||||
a/
|
||||
a/
|
||||
B
|
||||
b/
|
||||
`, nil, "", 2},
|
||||
{fs.DirEntries{
|
||||
mockobject.Object("a/b/c"),
|
||||
mockobject.Object("a/b/c/d/e"),
|
||||
}, `/
|
||||
a/b/
|
||||
`,
|
||||
level: 2,
|
||||
},
|
||||
{
|
||||
entries: fs.DirEntries{
|
||||
mockobject.Object("a/b/c"),
|
||||
mockobject.Object("a/b/c/d/e"),
|
||||
},
|
||||
want: `/
|
||||
a/
|
||||
a/
|
||||
b/
|
||||
`, nil, "", 2},
|
||||
a/b/
|
||||
`,
|
||||
level: 2,
|
||||
},
|
||||
{
|
||||
entries: fs.DirEntries{
|
||||
mockobject.Object("a/.bzEmpty"),
|
||||
mockobject.Object("a/b1/.bzEmpty"),
|
||||
mockobject.Object("a/b2/.bzEmpty"),
|
||||
},
|
||||
want: `/
|
||||
a/
|
||||
a/
|
||||
.bzEmpty
|
||||
b1/
|
||||
b2/
|
||||
a/b1/
|
||||
.bzEmpty
|
||||
a/b2/
|
||||
.bzEmpty
|
||||
`,
|
||||
level: -1,
|
||||
exclude: ""},
|
||||
{
|
||||
entries: fs.DirEntries{
|
||||
mockobject.Object("a/.bzEmpty"),
|
||||
mockobject.Object("a/b1/.bzEmpty"),
|
||||
mockobject.Object("a/b2/.bzEmpty"),
|
||||
},
|
||||
want: `/
|
||||
a/
|
||||
a/
|
||||
b1/
|
||||
b2/
|
||||
a/b1/
|
||||
a/b2/
|
||||
`,
|
||||
level: -1,
|
||||
exclude: ".bzEmpty",
|
||||
},
|
||||
} {
|
||||
r, err := walkRDirTree(context.Background(), nil, test.root, true, test.level, makeListRCallback(test.entries, test.err))
|
||||
assert.Equal(t, test.err, err, fmt.Sprintf("%+v", test))
|
||||
assert.Equal(t, test.want, r.String(), fmt.Sprintf("%+v", test))
|
||||
ctx := context.Background()
|
||||
if test.exclude != "" {
|
||||
fi, err := filter.NewFilter(nil)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, fi.Add(false, test.exclude))
|
||||
// Change the active filter
|
||||
ctx = filter.ReplaceConfig(ctx, fi)
|
||||
|
||||
}
|
||||
r, err := walkRDirTree(ctx, nil, test.root, test.exclude == "", test.level, makeListRCallback(test.entries, test.err))
|
||||
what := fmt.Sprintf("%+v", test)
|
||||
assert.Equal(t, test.err, err, what)
|
||||
assert.Equal(t, test.want, r.String(), what)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue