From c22d04aa309c606cec7a7ac248df2fb4ee5aae15 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 26 Nov 2020 17:10:41 +0000 Subject: [PATCH] filter: deglobalise to put filter config into the context #4685 --- cmd/cmd.go | 5 +-- cmd/serve/http/http_test.go | 6 ++-- cmd/serve/webdav/webdav_test.go | 6 ++-- fs/filter/filter.go | 41 +++++++++++++++++++++-- fs/filter/filter_test.go | 27 +++++++++++++++ fs/filter/filterflags/filterflags.go | 9 +++-- fs/list/list.go | 5 +-- fs/march/march.go | 6 ++-- fs/march/march_test.go | 6 ++-- fs/operations/listdirsorted_test.go | 10 +++--- fs/operations/operations_test.go | 13 +++----- fs/sync/sync.go | 8 +++-- fs/sync/sync_test.go | 39 +++++++++++----------- fs/walk/walk.go | 35 +++++++++++--------- fs/walk/walk_test.go | 49 ++++++++++++++-------------- 15 files changed, 176 insertions(+), 89 deletions(-) diff --git a/cmd/cmd.go b/cmd/cmd.go index 3d5224ff2..fd42f8891 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -109,15 +109,16 @@ func NewFsFile(remote string) (fs.Fs, string) { // This works the same as NewFsFile however it adds filters to the Fs // to limit it to a single file if the remote pointed to a file. func newFsFileAddFilter(remote string) (fs.Fs, string) { + fi := filter.GetConfig(context.Background()) f, fileName := NewFsFile(remote) if fileName != "" { - if !filter.Active.InActive() { + if !fi.InActive() { err := errors.Errorf("Can't limit to single files when using filters: %v", remote) err = fs.CountError(err) log.Fatalf(err.Error()) } // Limit transfers to this file - err := filter.Active.AddFile(fileName) + err := fi.AddFile(fileName) if err != nil { err = fs.CountError(err) log.Fatalf("Failed to limit to single file %q: %v", remote, err) diff --git a/cmd/serve/http/http_test.go b/cmd/serve/http/http_test.go index 4ace4ec2c..ed76fa879 100644 --- a/cmd/serve/http/http_test.go +++ b/cmd/serve/http/http_test.go @@ -59,6 +59,7 @@ var ( ) func TestInit(t *testing.T) { + ctx := context.Background() // Configure the remote config.LoadConfig(context.Background()) // fs.Config.LogLevel = fs.LogLevelDebug @@ -66,8 +67,9 @@ func TestInit(t *testing.T) { // fs.Config.DumpBodies = true // exclude files called hidden.txt and directories called hidden - require.NoError(t, filter.Active.AddRule("- hidden.txt")) - require.NoError(t, filter.Active.AddRule("- hidden/**")) + fi := filter.GetConfig(ctx) + require.NoError(t, fi.AddRule("- hidden.txt")) + require.NoError(t, fi.AddRule("- hidden/**")) // Create a test Fs f, err := fs.NewFs(context.Background(), "testdata/files") diff --git a/cmd/serve/webdav/webdav_test.go b/cmd/serve/webdav/webdav_test.go index 32c50abbd..cef20546d 100644 --- a/cmd/serve/webdav/webdav_test.go +++ b/cmd/serve/webdav/webdav_test.go @@ -87,9 +87,11 @@ var ( ) func TestHTTPFunction(t *testing.T) { + ctx := context.Background() // exclude files called hidden.txt and directories called hidden - require.NoError(t, filter.Active.AddRule("- hidden.txt")) - require.NoError(t, filter.Active.AddRule("- hidden/**")) + fi := filter.GetConfig(ctx) + require.NoError(t, fi.AddRule("- hidden.txt")) + require.NoError(t, fi.AddRule("- hidden/**")) // Uses the same test files as http tests but with different golden. f, err := fs.NewFs(context.Background(), "../http/testdata/files") diff --git a/fs/filter/filter.go b/fs/filter/filter.go index f39c1f71b..1714dabf5 100644 --- a/fs/filter/filter.go +++ b/fs/filter/filter.go @@ -17,8 +17,10 @@ import ( "golang.org/x/sync/errgroup" ) -// Active is the globally active filter -var Active = mustNewFilter(nil) +// This is the globally active filter +// +// This is accessed through GetConfig and AddConfig +var globalConfig = mustNewFilter(nil) // rule is one filter rule type rule struct { @@ -591,3 +593,38 @@ func (f *Filter) UsesDirectoryFilters() bool { } return true } + +type configContextKeyType struct{} + +// Context key for config +var configContextKey = configContextKeyType{} + +// GetConfig returns the global or context sensitive config +func GetConfig(ctx context.Context) *Filter { + if ctx == nil { + return globalConfig + } + c := ctx.Value(configContextKey) + if c == nil { + return globalConfig + } + return c.(*Filter) +} + +// AddConfig returns a mutable config structure based on a shallow +// copy of that found in ctx and returns a new context with that added +// to it. +func AddConfig(ctx context.Context) (context.Context, *Filter) { + c := GetConfig(ctx) + cCopy := new(Filter) + *cCopy = *c + newCtx := context.WithValue(ctx, configContextKey, cCopy) + return newCtx, cCopy +} + +// ReplaceConfig replaces the filter config in the ctx with the one +// passed in and returns a new context with that added to it. +func ReplaceConfig(ctx context.Context, f *Filter) context.Context { + newCtx := context.WithValue(ctx, configContextKey, f) + return newCtx +} diff --git a/fs/filter/filter_test.go b/fs/filter/filter_test.go index 399d4616e..1fc4ef671 100644 --- a/fs/filter/filter_test.go +++ b/fs/filter/filter_test.go @@ -753,3 +753,30 @@ func TestNewFilterUsesDirectoryFilters(t *testing.T) { assert.Equal(t, test.want, got, fmt.Sprintf("%s: %s", what, f.DumpFilters())) } } + +func TestGetConfig(t *testing.T) { + ctx := context.Background() + + // Check nil + config := GetConfig(nil) + assert.Equal(t, globalConfig, config) + + // Check empty config + config = GetConfig(ctx) + assert.Equal(t, globalConfig, config) + + // Check adding a config + ctx2, config2 := AddConfig(ctx) + require.NoError(t, config2.AddRule("+ *.jpg")) + assert.NotEqual(t, config2, config) + + // Check can get config back + config2ctx := GetConfig(ctx2) + assert.Equal(t, config2, config2ctx) + + // Check ReplaceConfig + f, err := NewFilter(nil) + require.NoError(t, err) + ctx3 := ReplaceConfig(ctx, f) + assert.Equal(t, globalConfig, GetConfig(ctx3)) +} diff --git a/fs/filter/filterflags/filterflags.go b/fs/filter/filterflags/filterflags.go index f37574cb2..3a3d3e4c7 100644 --- a/fs/filter/filterflags/filterflags.go +++ b/fs/filter/filterflags/filterflags.go @@ -17,8 +17,13 @@ var ( // Reload the filters from the flags func Reload(ctx context.Context) (err error) { - filter.Active, err = filter.NewFilter(&Opt) - return err + fi := filter.GetConfig(ctx) + newFilter, err := filter.NewFilter(&Opt) + if err != nil { + return err + } + *fi = *newFilter + return nil } // AddFlags adds the non filing system specific flags to the command diff --git a/fs/list/list.go b/fs/list/list.go index 7e33c7517..dfa8b688d 100644 --- a/fs/list/list.go +++ b/fs/list/list.go @@ -28,11 +28,12 @@ func DirSorted(ctx context.Context, f fs.Fs, includeAll bool, dir string) (entri // This should happen only if exclude files lives in the // starting directory, otherwise ListDirSorted should not be // called. - if !includeAll && filter.Active.ListContainsExcludeFile(entries) { + fi := filter.GetConfig(ctx) + if !includeAll && fi.ListContainsExcludeFile(entries) { fs.Debugf(dir, "Excluded") return nil, nil } - return filterAndSortDir(ctx, entries, includeAll, dir, filter.Active.IncludeObject, filter.Active.IncludeDirectory(ctx, f)) + return filterAndSortDir(ctx, entries, includeAll, dir, fi.IncludeObject, fi.IncludeDirectory(ctx, f)) } // filter (if required) and check the entries, then sort them diff --git a/fs/march/march.go b/fs/march/march.go index 7307cc8f6..3e363a52f 100644 --- a/fs/march/march.go +++ b/fs/march/march.go @@ -78,8 +78,9 @@ type listDirFn func(dir string) (entries fs.DirEntries, err error) // and includeAll flags for marching through the file system. func (m *March) makeListDir(ctx context.Context, f fs.Fs, includeAll bool) listDirFn { ci := fs.GetConfig(ctx) + fi := filter.GetConfig(ctx) if !(ci.UseListR && f.Features().ListR != nil) && // !--fast-list active and - !(ci.NoTraverse && filter.Active.HaveFilesFrom()) { // !(--files-from and --no-traverse) + !(ci.NoTraverse && fi.HaveFilesFrom()) { // !(--files-from and --no-traverse) return func(dir string) (entries fs.DirEntries, err error) { return list.DirSorted(m.Ctx, f, includeAll, dir) } @@ -126,6 +127,7 @@ type listDirJob struct { // Run starts the matching process off func (m *March) Run(ctx context.Context) error { ci := fs.GetConfig(ctx) + fi := filter.GetConfig(ctx) m.init(ctx) srcDepth := ci.MaxDepth @@ -133,7 +135,7 @@ func (m *March) Run(ctx context.Context) error { srcDepth = fs.MaxLevel } dstDepth := srcDepth - if filter.Active.Opt.DeleteExcluded { + if fi.Opt.DeleteExcluded { dstDepth = fs.MaxLevel } diff --git a/fs/march/march_test.go b/fs/march/march_test.go index 3d1a58040..31814255e 100644 --- a/fs/march/march_test.go +++ b/fs/march/march_test.go @@ -193,6 +193,7 @@ func TestMarch(t *testing.T) { cancel: cancel, noTraverse: false, } + fi := filter.GetConfig(ctx) m := &March{ Ctx: ctx, Fdst: r.Fremote, @@ -200,7 +201,7 @@ func TestMarch(t *testing.T) { Dir: "", NoTraverse: mt.noTraverse, Callback: mt, - DstIncludeAll: filter.Active.Opt.DeleteExcluded, + DstIncludeAll: fi.Opt.DeleteExcluded, } mt.processError(m.Run(ctx)) @@ -260,6 +261,7 @@ func TestMarchNoTraverse(t *testing.T) { cancel: cancel, noTraverse: true, } + fi := filter.GetConfig(ctx) m := &March{ Ctx: ctx, Fdst: r.Fremote, @@ -267,7 +269,7 @@ func TestMarchNoTraverse(t *testing.T) { Dir: "", NoTraverse: mt.noTraverse, Callback: mt, - DstIncludeAll: filter.Active.Opt.DeleteExcluded, + DstIncludeAll: fi.Opt.DeleteExcluded, } mt.processError(m.Run(ctx)) diff --git a/fs/operations/listdirsorted_test.go b/fs/operations/listdirsorted_test.go index 708417e9f..d15732c43 100644 --- a/fs/operations/listdirsorted_test.go +++ b/fs/operations/listdirsorted_test.go @@ -18,9 +18,11 @@ func TestListDirSorted(t *testing.T) { r := fstest.NewRun(t) defer r.Finalise() - filter.Active.Opt.MaxSize = 10 + ctx := context.Background() + fi := filter.GetConfig(ctx) + fi.Opt.MaxSize = 10 defer func() { - filter.Active.Opt.MaxSize = -1 + fi.Opt.MaxSize = -1 }() files := []fstest.Item{ @@ -79,7 +81,7 @@ func TestListDirSorted(t *testing.T) { assert.Equal(t, "sub dir/sub sub dir/", str(1)) // testing ignore file - filter.Active.Opt.ExcludeFile = ".ignore" + fi.Opt.ExcludeFile = ".ignore" items, err = list.DirSorted(context.Background(), r.Fremote, false, "sub dir") require.NoError(t, err) @@ -96,7 +98,7 @@ func TestListDirSorted(t *testing.T) { assert.Equal(t, "sub dir/ignore dir/.ignore", str(0)) assert.Equal(t, "sub dir/ignore dir/should be ignored", str(1)) - filter.Active.Opt.ExcludeFile = "" + fi.Opt.ExcludeFile = "" items, err = list.DirSorted(context.Background(), r.Fremote, false, "sub dir/ignore dir") require.NoError(t, err) require.Len(t, items, 2) diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go index 113e7a198..e5f30db55 100644 --- a/fs/operations/operations_test.go +++ b/fs/operations/operations_test.go @@ -120,12 +120,8 @@ func TestLsWithFilesFrom(t *testing.T) { require.NoError(t, f.AddFile("potato2")) require.NoError(t, f.AddFile("notfound")) - // Monkey patch the active filter - oldFilter := filter.Active - filter.Active = f - defer func() { - filter.Active = oldFilter - }() + // Change the active filter + ctx = filter.ReplaceConfig(ctx, f) var buf bytes.Buffer err = operations.List(ctx, r.Fremote, &buf) @@ -321,6 +317,7 @@ func TestCount(t *testing.T) { func TestDelete(t *testing.T) { ctx := context.Background() + fi := filter.GetConfig(ctx) r := fstest.NewRun(t) defer r.Finalise() file1 := r.WriteObject(ctx, "small", "1234567890", t2) // 10 bytes @@ -328,9 +325,9 @@ func TestDelete(t *testing.T) { file3 := r.WriteObject(ctx, "large", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 100 bytes fstest.CheckItems(t, r.Fremote, file1, file2, file3) - filter.Active.Opt.MaxSize = 60 + fi.Opt.MaxSize = 60 defer func() { - filter.Active.Opt.MaxSize = -1 + fi.Opt.MaxSize = -1 }() err := operations.Delete(ctx, r.Fremote) diff --git a/fs/sync/sync.go b/fs/sync/sync.go index 1eaaafadb..c3c209e27 100644 --- a/fs/sync/sync.go +++ b/fs/sync/sync.go @@ -31,6 +31,7 @@ type syncCopyMove struct { dir string // internal state ci *fs.ConfigInfo // global config + fi *filter.Filter // filter config ctx context.Context // internal context for controlling go-routines cancel func() // cancel the context inCtx context.Context // internal context for controlling march @@ -99,8 +100,10 @@ func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete return nil, fserrors.FatalError(fs.ErrorOverlapping) } ci := fs.GetConfig(ctx) + fi := filter.GetConfig(ctx) s := &syncCopyMove{ ci: ci, + fi: fi, fdst: fdst, fsrc: fsrc, deleteMode: deleteMode, @@ -828,7 +831,7 @@ func (s *syncCopyMove) run() error { Dir: s.dir, NoTraverse: s.noTraverse, Callback: s, - DstIncludeAll: filter.Active.Opt.DeleteExcluded, + DstIncludeAll: s.fi.Opt.DeleteExcluded, NoCheckDest: s.noCheckDest, NoUnicodeNormalization: s.noUnicodeNormalization, } @@ -1087,13 +1090,14 @@ func moveDir(ctx context.Context, fdst, fsrc fs.Fs, deleteEmptySrcDirs bool, cop // MoveDir moves fsrc into fdst func MoveDir(ctx context.Context, fdst, fsrc fs.Fs, deleteEmptySrcDirs bool, copyEmptySrcDirs bool) error { + fi := filter.GetConfig(ctx) if operations.Same(fdst, fsrc) { fs.Errorf(fdst, "Nothing to do as source and destination are the same") return nil } // First attempt to use DirMover if exists, same Fs and no filters are active - if fdstDirMove := fdst.Features().DirMove; fdstDirMove != nil && operations.SameConfig(fsrc, fdst) && filter.Active.InActive() { + if fdstDirMove := fdst.Features().DirMove; fdstDirMove != nil && operations.SameConfig(fsrc, fdst) && fi.InActive() { if operations.SkipDestructive(ctx, fdst, "server-side directory move") { return nil } diff --git a/fs/sync/sync_test.go b/fs/sync/sync_test.go index bee41dd22..92bed0d7a 100644 --- a/fs/sync/sync_test.go +++ b/fs/sync/sync_test.go @@ -177,13 +177,12 @@ func testCopyWithFilesFrom(t *testing.T, noTraverse bool) { require.NoError(t, f.AddFile("potato2")) require.NoError(t, f.AddFile("notfound")) - // Monkey patch the active filter - oldFilter := filter.Active + // Change the active filter + ctx = filter.ReplaceConfig(ctx, f) + oldNoTraverse := ci.NoTraverse - filter.Active = f ci.NoTraverse = noTraverse unpatch := func() { - filter.Active = oldFilter ci.NoTraverse = oldNoTraverse } defer unpatch() @@ -967,9 +966,10 @@ func TestSyncWithExclude(t *testing.T) { fstest.CheckItems(t, r.Fremote, file1, file2) fstest.CheckItems(t, r.Flocal, file1, file2, file3) - filter.Active.Opt.MaxSize = 40 + fi := filter.GetConfig(ctx) + fi.Opt.MaxSize = 40 defer func() { - filter.Active.Opt.MaxSize = -1 + fi.Opt.MaxSize = -1 }() accounting.GlobalStats().ResetCounters() @@ -996,11 +996,12 @@ func TestSyncWithExcludeAndDeleteExcluded(t *testing.T) { fstest.CheckItems(t, r.Fremote, file1, file2, file3) fstest.CheckItems(t, r.Flocal, file1, file2, file3) - filter.Active.Opt.MaxSize = 40 - filter.Active.Opt.DeleteExcluded = true + fi := filter.GetConfig(ctx) + fi.Opt.MaxSize = 40 + fi.Opt.DeleteExcluded = true defer func() { - filter.Active.Opt.MaxSize = -1 - filter.Active.Opt.DeleteExcluded = false + fi.Opt.MaxSize = -1 + fi.Opt.DeleteExcluded = false }() accounting.GlobalStats().ResetCounters() @@ -1399,12 +1400,14 @@ func TestServerSideMove(t *testing.T) { // Test a server-side move if possible, or the backup path if not func TestServerSideMoveWithFilter(t *testing.T) { + ctx := context.Background() r := fstest.NewRun(t) defer r.Finalise() - filter.Active.Opt.MinSize = 40 + fi := filter.GetConfig(ctx) + fi.Opt.MinSize = 40 defer func() { - filter.Active.Opt.MinSize = -1 + fi.Opt.MinSize = -1 }() testServerSideMove(t, r, true, false) @@ -1439,9 +1442,10 @@ func TestServerSideMoveOverlap(t *testing.T) { assert.EqualError(t, err, fs.ErrorOverlapping.Error()) // Now try with a filter which should also fail with ErrorCantMoveOverlapping - filter.Active.Opt.MinSize = 40 + fi := filter.GetConfig(ctx) + fi.Opt.MinSize = 40 defer func() { - filter.Active.Opt.MinSize = -1 + fi.Opt.MinSize = -1 }() err = MoveDir(ctx, FremoteMove, r.Fremote, false, false) assert.EqualError(t, err, fs.ErrorOverlapping.Error()) @@ -1686,11 +1690,8 @@ func testSyncBackupDir(t *testing.T, backupDir string, suffix string, suffixKeep flt, err := filter.NewFilter(nil) require.NoError(t, err) require.NoError(t, flt.AddRule("- *"+suffix)) - oldFlt := filter.Active - filter.Active = flt - defer func() { - filter.Active = oldFlt - }() + // Change the active filter + ctx = filter.ReplaceConfig(ctx, flt) } ci.Suffix = suffix ci.SuffixKeepExtension = suffixKeepExtension diff --git a/fs/walk/walk.go b/fs/walk/walk.go index 04510e053..b731fcf5d 100644 --- a/fs/walk/walk.go +++ b/fs/walk/walk.go @@ -60,8 +60,9 @@ type Func func(path string, entries fs.DirEntries, err error) error // NB (f, path) to be replaced by fs.Dir at some point func Walk(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel int, fn Func) error { ci := fs.GetConfig(ctx) - if ci.NoTraverse && filter.Active.HaveFilesFrom() { - return walkR(ctx, f, path, includeAll, maxLevel, fn, filter.Active.MakeListR(ctx, f.NewObject)) + fi := filter.GetConfig(ctx) + if ci.NoTraverse && fi.HaveFilesFrom() { + return walkR(ctx, f, path, includeAll, maxLevel, fn, fi.MakeListR(ctx, f.NewObject)) } // FIXME should this just be maxLevel < 0 - why the maxLevel > 1 if (maxLevel < 0 || maxLevel > 1) && ci.UseListR && f.Features().ListR != nil { @@ -139,15 +140,16 @@ func (l ListType) Filter(in *fs.DirEntries) { // // NB (f, path) to be replaced by fs.Dir at some point func ListR(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel int, listType ListType, fn fs.ListRCallback) error { + fi := filter.GetConfig(ctx) // FIXME disable this with --no-fast-list ??? `--disable ListR` will do it... doListR := f.Features().ListR // Can't use ListR if... if doListR == nil || // ...no ListR - filter.Active.HaveFilesFrom() || // ...using --files-from + fi.HaveFilesFrom() || // ...using --files-from maxLevel >= 0 || // ...using bounded recursion - len(filter.Active.Opt.ExcludeFile) > 0 || // ...using --exclude-file - filter.Active.UsesDirectoryFilters() { // ...using any directory filters + len(fi.Opt.ExcludeFile) > 0 || // ...using --exclude-file + fi.UsesDirectoryFilters() { // ...using any directory filters return listRwalk(ctx, f, path, includeAll, maxLevel, listType, fn) } return listR(ctx, f, path, includeAll, listType, fn, doListR, listType.Dirs() && f.Features().BucketBased) @@ -275,9 +277,10 @@ func (dm *dirMap) sendEntries(fn fs.ListRCallback) (err error) { // listR walks the file tree using ListR func listR(ctx context.Context, f fs.Fs, path string, includeAll bool, listType ListType, fn fs.ListRCallback, doListR fs.ListRFn, synthesizeDirs bool) error { - includeDirectory := filter.Active.IncludeDirectory(ctx, f) + fi := filter.GetConfig(ctx) + includeDirectory := fi.IncludeDirectory(ctx, f) if !includeAll { - includeAll = filter.Active.InActive() + includeAll = fi.InActive() } var dm *dirMap if synthesizeDirs { @@ -298,7 +301,7 @@ func listR(ctx context.Context, f fs.Fs, path string, includeAll bool, listType var include bool switch x := entry.(type) { case fs.Object: - include = filter.Active.IncludeObject(ctx, x) + include = fi.IncludeObject(ctx, x) case fs.Directory: include, err = includeDirectory(x.Remote()) if err != nil { @@ -448,11 +451,12 @@ func walk(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel i } func walkRDirTree(ctx context.Context, f fs.Fs, startPath string, includeAll bool, maxLevel int, listR fs.ListRFn) (dirtree.DirTree, error) { + fi := filter.GetConfig(ctx) dirs := dirtree.New() // Entries can come in arbitrary order. We use toPrune to keep // all directories to exclude later. toPrune := make(map[string]bool) - includeDirectory := filter.Active.IncludeDirectory(ctx, f) + includeDirectory := fi.IncludeDirectory(ctx, f) var mu sync.Mutex err := listR(ctx, startPath, func(entries fs.DirEntries) error { mu.Lock() @@ -462,7 +466,7 @@ func walkRDirTree(ctx context.Context, f fs.Fs, startPath string, includeAll boo switch x := entry.(type) { case fs.Object: // Make sure we don't delete excluded files if not required - if includeAll || filter.Active.IncludeObject(ctx, x) { + if includeAll || fi.IncludeObject(ctx, x) { if maxLevel < 0 || slashes <= maxLevel-1 { dirs.Add(x) } else { @@ -477,9 +481,9 @@ func walkRDirTree(ctx context.Context, f fs.Fs, startPath string, includeAll boo fs.Debugf(x, "Excluded from sync (and deletion)") } // Check if we need to prune a directory later. - if !includeAll && len(filter.Active.Opt.ExcludeFile) > 0 { + if !includeAll && len(fi.Opt.ExcludeFile) > 0 { basename := path.Base(x.Remote()) - if basename == filter.Active.Opt.ExcludeFile { + if basename == fi.Opt.ExcludeFile { excludeDir := parentDir(x.Remote()) toPrune[excludeDir] = true fs.Debugf(basename, "Excluded from sync (and deletion) based on exclude file") @@ -556,12 +560,13 @@ func walkNDirTree(ctx context.Context, f fs.Fs, path string, includeAll bool, ma // NB (f, path) to be replaced by fs.Dir at some point func NewDirTree(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel int) (dirtree.DirTree, error) { ci := fs.GetConfig(ctx) + fi := filter.GetConfig(ctx) // if --no-traverse and --files-from build DirTree just from files - if ci.NoTraverse && filter.Active.HaveFilesFrom() { - return walkRDirTree(ctx, f, path, includeAll, maxLevel, filter.Active.MakeListR(ctx, f.NewObject)) + if ci.NoTraverse && fi.HaveFilesFrom() { + return walkRDirTree(ctx, f, path, includeAll, maxLevel, fi.MakeListR(ctx, f.NewObject)) } // if have ListR; and recursing; and not using --files-from; then build a DirTree with ListR - if ListR := f.Features().ListR; (maxLevel < 0 || maxLevel > 1) && ListR != nil && !filter.Active.HaveFilesFrom() { + if ListR := f.Features().ListR; (maxLevel < 0 || maxLevel > 1) && ListR != nil && !fi.HaveFilesFrom() { return walkRDirTree(ctx, f, path, includeAll, maxLevel, ListR) } // otherwise just use List diff --git a/fs/walk/walk_test.go b/fs/walk/walk_test.go index 509797b0f..b670ef2cb 100644 --- a/fs/walk/walk_test.go +++ b/fs/walk/walk_test.go @@ -585,6 +585,8 @@ a/ } func TestWalkRDirTreeExclude(t *testing.T) { + ctx := context.Background() + fi := filter.GetConfig(ctx) for _, test := range []struct { entries fs.DirEntries want string @@ -648,13 +650,13 @@ b/c/d/ e `, nil, "", -1, "ign", true}, } { - filter.Active.Opt.ExcludeFile = test.excludeFile + fi.Opt.ExcludeFile = test.excludeFile r, err := walkRDirTree(context.Background(), nil, test.root, test.includeAll, test.level, makeListRCallback(test.entries, test.err)) assert.Equal(t, test.err, err, fmt.Sprintf("%+v", test)) assert.Equal(t, test.want, r.String(), fmt.Sprintf("%+v", test)) } // Set to default value, to avoid side effects - filter.Active.Opt.ExcludeFile = "" + fi.Opt.ExcludeFile = "" } func TestListType(t *testing.T) { @@ -701,6 +703,7 @@ func TestListType(t *testing.T) { } func TestListR(t *testing.T) { + ctx := context.Background() objects := fs.DirEntries{ mockobject.Object("a"), mockobject.Object("b"), @@ -709,7 +712,7 @@ func TestListR(t *testing.T) { mockobject.Object("dir/b"), mockobject.Object("dir/c"), } - f := mockfs.NewFs(context.Background(), "mock", "/") + f := mockfs.NewFs(ctx, "mock", "/") var got []string clearCallback := func() { got = nil @@ -730,57 +733,53 @@ func TestListR(t *testing.T) { return callback(os) } - // Setup filter - oldFilter := filter.Active - defer func() { - filter.Active = oldFilter - }() - - var err error - filter.Active, err = filter.NewFilter(nil) + fi, err := filter.NewFilter(nil) require.NoError(t, err) - require.NoError(t, filter.Active.AddRule("+ b")) - require.NoError(t, filter.Active.AddRule("- *")) + require.NoError(t, fi.AddRule("+ b")) + require.NoError(t, fi.AddRule("- *")) + + // Change the active filter + ctx = filter.ReplaceConfig(ctx, fi) // Base case clearCallback() - err = listR(context.Background(), f, "", true, ListAll, callback, doListR, false) + err = listR(ctx, f, "", true, ListAll, callback, doListR, false) require.NoError(t, err) require.Equal(t, []string{"a", "b", "dir", "dir/a", "dir/b", "dir/c"}, got) // Base case - with Objects clearCallback() - err = listR(context.Background(), f, "", true, ListObjects, callback, doListR, false) + err = listR(ctx, f, "", true, ListObjects, callback, doListR, false) require.NoError(t, err) require.Equal(t, []string{"a", "b", "dir/a", "dir/b", "dir/c"}, got) // Base case - with Dirs clearCallback() - err = listR(context.Background(), f, "", true, ListDirs, callback, doListR, false) + err = listR(ctx, f, "", true, ListDirs, callback, doListR, false) require.NoError(t, err) require.Equal(t, []string{"dir"}, got) // With filter clearCallback() - err = listR(context.Background(), f, "", false, ListAll, callback, doListR, false) + err = listR(ctx, f, "", false, ListAll, callback, doListR, false) require.NoError(t, err) require.Equal(t, []string{"b", "dir", "dir/b"}, got) // With filter - with Objects clearCallback() - err = listR(context.Background(), f, "", false, ListObjects, callback, doListR, false) + err = listR(ctx, f, "", false, ListObjects, callback, doListR, false) require.NoError(t, err) require.Equal(t, []string{"b", "dir/b"}, got) // With filter - with Dir clearCallback() - err = listR(context.Background(), f, "", false, ListDirs, callback, doListR, false) + err = listR(ctx, f, "", false, ListDirs, callback, doListR, false) require.NoError(t, err) require.Equal(t, []string{"dir"}, got) // With filter and subdir clearCallback() - err = listR(context.Background(), f, "dir", false, ListAll, callback, doListR, false) + err = listR(ctx, f, "dir", false, ListAll, callback, doListR, false) require.NoError(t, err) require.Equal(t, []string{"dir/b"}, got) @@ -796,31 +795,31 @@ func TestListR(t *testing.T) { // Base case clearCallback() - err = listR(context.Background(), f, "", true, ListAll, callback, doListR, true) + err = listR(ctx, f, "", true, ListAll, callback, doListR, true) require.NoError(t, err) require.Equal(t, []string{"a", "b", "dir/a", "dir/b", "dir/subdir/c", "dir/subdir", "dir"}, got) // With filter clearCallback() - err = listR(context.Background(), f, "", false, ListAll, callback, doListR, true) + err = listR(ctx, f, "", false, ListAll, callback, doListR, true) require.NoError(t, err) require.Equal(t, []string{"b", "dir/b", "dir/subdir", "dir"}, got) // With filter and subdir clearCallback() - err = listR(context.Background(), f, "dir", false, ListAll, callback, doListR, true) + err = listR(ctx, f, "dir", false, ListAll, callback, doListR, true) require.NoError(t, err) require.Equal(t, []string{"dir/b", "dir/subdir"}, got) // With filter and subdir - with Objects clearCallback() - err = listR(context.Background(), f, "dir", false, ListObjects, callback, doListR, true) + err = listR(ctx, f, "dir", false, ListObjects, callback, doListR, true) require.NoError(t, err) require.Equal(t, []string{"dir/b"}, got) // With filter and subdir - with Dirs clearCallback() - err = listR(context.Background(), f, "dir", false, ListDirs, callback, doListR, true) + err = listR(ctx, f, "dir", false, ListDirs, callback, doListR, true) require.NoError(t, err) require.Equal(t, []string{"dir/subdir"}, got) }