filter: deglobalise to put filter config into the context #4685

This commit is contained in:
Nick Craig-Wood 2020-11-26 17:10:41 +00:00
parent 354b4f19ec
commit c22d04aa30
15 changed files with 176 additions and 89 deletions

View File

@ -109,15 +109,16 @@ func NewFsFile(remote string) (fs.Fs, string) {
// This works the same as NewFsFile however it adds filters to the Fs // This works the same as NewFsFile however it adds filters to the Fs
// to limit it to a single file if the remote pointed to a file. // to limit it to a single file if the remote pointed to a file.
func newFsFileAddFilter(remote string) (fs.Fs, string) { func newFsFileAddFilter(remote string) (fs.Fs, string) {
fi := filter.GetConfig(context.Background())
f, fileName := NewFsFile(remote) f, fileName := NewFsFile(remote)
if fileName != "" { if fileName != "" {
if !filter.Active.InActive() { if !fi.InActive() {
err := errors.Errorf("Can't limit to single files when using filters: %v", remote) err := errors.Errorf("Can't limit to single files when using filters: %v", remote)
err = fs.CountError(err) err = fs.CountError(err)
log.Fatalf(err.Error()) log.Fatalf(err.Error())
} }
// Limit transfers to this file // Limit transfers to this file
err := filter.Active.AddFile(fileName) err := fi.AddFile(fileName)
if err != nil { if err != nil {
err = fs.CountError(err) err = fs.CountError(err)
log.Fatalf("Failed to limit to single file %q: %v", remote, err) log.Fatalf("Failed to limit to single file %q: %v", remote, err)

View File

@ -59,6 +59,7 @@ var (
) )
func TestInit(t *testing.T) { func TestInit(t *testing.T) {
ctx := context.Background()
// Configure the remote // Configure the remote
config.LoadConfig(context.Background()) config.LoadConfig(context.Background())
// fs.Config.LogLevel = fs.LogLevelDebug // fs.Config.LogLevel = fs.LogLevelDebug
@ -66,8 +67,9 @@ func TestInit(t *testing.T) {
// fs.Config.DumpBodies = true // fs.Config.DumpBodies = true
// exclude files called hidden.txt and directories called hidden // exclude files called hidden.txt and directories called hidden
require.NoError(t, filter.Active.AddRule("- hidden.txt")) fi := filter.GetConfig(ctx)
require.NoError(t, filter.Active.AddRule("- hidden/**")) require.NoError(t, fi.AddRule("- hidden.txt"))
require.NoError(t, fi.AddRule("- hidden/**"))
// Create a test Fs // Create a test Fs
f, err := fs.NewFs(context.Background(), "testdata/files") f, err := fs.NewFs(context.Background(), "testdata/files")

View File

@ -87,9 +87,11 @@ var (
) )
func TestHTTPFunction(t *testing.T) { func TestHTTPFunction(t *testing.T) {
ctx := context.Background()
// exclude files called hidden.txt and directories called hidden // exclude files called hidden.txt and directories called hidden
require.NoError(t, filter.Active.AddRule("- hidden.txt")) fi := filter.GetConfig(ctx)
require.NoError(t, filter.Active.AddRule("- hidden/**")) require.NoError(t, fi.AddRule("- hidden.txt"))
require.NoError(t, fi.AddRule("- hidden/**"))
// Uses the same test files as http tests but with different golden. // Uses the same test files as http tests but with different golden.
f, err := fs.NewFs(context.Background(), "../http/testdata/files") f, err := fs.NewFs(context.Background(), "../http/testdata/files")

View File

@ -17,8 +17,10 @@ import (
"golang.org/x/sync/errgroup" "golang.org/x/sync/errgroup"
) )
// Active is the globally active filter // This is the globally active filter
var Active = mustNewFilter(nil) //
// This is accessed through GetConfig and AddConfig
var globalConfig = mustNewFilter(nil)
// rule is one filter rule // rule is one filter rule
type rule struct { type rule struct {
@ -591,3 +593,38 @@ func (f *Filter) UsesDirectoryFilters() bool {
} }
return true return true
} }
type configContextKeyType struct{}
// Context key for config
var configContextKey = configContextKeyType{}
// GetConfig returns the global or context sensitive config
func GetConfig(ctx context.Context) *Filter {
if ctx == nil {
return globalConfig
}
c := ctx.Value(configContextKey)
if c == nil {
return globalConfig
}
return c.(*Filter)
}
// AddConfig returns a mutable config structure based on a shallow
// copy of that found in ctx and returns a new context with that added
// to it.
func AddConfig(ctx context.Context) (context.Context, *Filter) {
c := GetConfig(ctx)
cCopy := new(Filter)
*cCopy = *c
newCtx := context.WithValue(ctx, configContextKey, cCopy)
return newCtx, cCopy
}
// ReplaceConfig replaces the filter config in the ctx with the one
// passed in and returns a new context with that added to it.
func ReplaceConfig(ctx context.Context, f *Filter) context.Context {
newCtx := context.WithValue(ctx, configContextKey, f)
return newCtx
}

View File

@ -753,3 +753,30 @@ func TestNewFilterUsesDirectoryFilters(t *testing.T) {
assert.Equal(t, test.want, got, fmt.Sprintf("%s: %s", what, f.DumpFilters())) assert.Equal(t, test.want, got, fmt.Sprintf("%s: %s", what, f.DumpFilters()))
} }
} }
func TestGetConfig(t *testing.T) {
ctx := context.Background()
// Check nil
config := GetConfig(nil)
assert.Equal(t, globalConfig, config)
// Check empty config
config = GetConfig(ctx)
assert.Equal(t, globalConfig, config)
// Check adding a config
ctx2, config2 := AddConfig(ctx)
require.NoError(t, config2.AddRule("+ *.jpg"))
assert.NotEqual(t, config2, config)
// Check can get config back
config2ctx := GetConfig(ctx2)
assert.Equal(t, config2, config2ctx)
// Check ReplaceConfig
f, err := NewFilter(nil)
require.NoError(t, err)
ctx3 := ReplaceConfig(ctx, f)
assert.Equal(t, globalConfig, GetConfig(ctx3))
}

View File

@ -17,8 +17,13 @@ var (
// Reload the filters from the flags // Reload the filters from the flags
func Reload(ctx context.Context) (err error) { func Reload(ctx context.Context) (err error) {
filter.Active, err = filter.NewFilter(&Opt) fi := filter.GetConfig(ctx)
return err newFilter, err := filter.NewFilter(&Opt)
if err != nil {
return err
}
*fi = *newFilter
return nil
} }
// AddFlags adds the non filing system specific flags to the command // AddFlags adds the non filing system specific flags to the command

View File

@ -28,11 +28,12 @@ func DirSorted(ctx context.Context, f fs.Fs, includeAll bool, dir string) (entri
// This should happen only if exclude files lives in the // This should happen only if exclude files lives in the
// starting directory, otherwise ListDirSorted should not be // starting directory, otherwise ListDirSorted should not be
// called. // called.
if !includeAll && filter.Active.ListContainsExcludeFile(entries) { fi := filter.GetConfig(ctx)
if !includeAll && fi.ListContainsExcludeFile(entries) {
fs.Debugf(dir, "Excluded") fs.Debugf(dir, "Excluded")
return nil, nil return nil, nil
} }
return filterAndSortDir(ctx, entries, includeAll, dir, filter.Active.IncludeObject, filter.Active.IncludeDirectory(ctx, f)) return filterAndSortDir(ctx, entries, includeAll, dir, fi.IncludeObject, fi.IncludeDirectory(ctx, f))
} }
// filter (if required) and check the entries, then sort them // filter (if required) and check the entries, then sort them

View File

@ -78,8 +78,9 @@ type listDirFn func(dir string) (entries fs.DirEntries, err error)
// and includeAll flags for marching through the file system. // and includeAll flags for marching through the file system.
func (m *March) makeListDir(ctx context.Context, f fs.Fs, includeAll bool) listDirFn { func (m *March) makeListDir(ctx context.Context, f fs.Fs, includeAll bool) listDirFn {
ci := fs.GetConfig(ctx) ci := fs.GetConfig(ctx)
fi := filter.GetConfig(ctx)
if !(ci.UseListR && f.Features().ListR != nil) && // !--fast-list active and if !(ci.UseListR && f.Features().ListR != nil) && // !--fast-list active and
!(ci.NoTraverse && filter.Active.HaveFilesFrom()) { // !(--files-from and --no-traverse) !(ci.NoTraverse && fi.HaveFilesFrom()) { // !(--files-from and --no-traverse)
return func(dir string) (entries fs.DirEntries, err error) { return func(dir string) (entries fs.DirEntries, err error) {
return list.DirSorted(m.Ctx, f, includeAll, dir) return list.DirSorted(m.Ctx, f, includeAll, dir)
} }
@ -126,6 +127,7 @@ type listDirJob struct {
// Run starts the matching process off // Run starts the matching process off
func (m *March) Run(ctx context.Context) error { func (m *March) Run(ctx context.Context) error {
ci := fs.GetConfig(ctx) ci := fs.GetConfig(ctx)
fi := filter.GetConfig(ctx)
m.init(ctx) m.init(ctx)
srcDepth := ci.MaxDepth srcDepth := ci.MaxDepth
@ -133,7 +135,7 @@ func (m *March) Run(ctx context.Context) error {
srcDepth = fs.MaxLevel srcDepth = fs.MaxLevel
} }
dstDepth := srcDepth dstDepth := srcDepth
if filter.Active.Opt.DeleteExcluded { if fi.Opt.DeleteExcluded {
dstDepth = fs.MaxLevel dstDepth = fs.MaxLevel
} }

View File

@ -193,6 +193,7 @@ func TestMarch(t *testing.T) {
cancel: cancel, cancel: cancel,
noTraverse: false, noTraverse: false,
} }
fi := filter.GetConfig(ctx)
m := &March{ m := &March{
Ctx: ctx, Ctx: ctx,
Fdst: r.Fremote, Fdst: r.Fremote,
@ -200,7 +201,7 @@ func TestMarch(t *testing.T) {
Dir: "", Dir: "",
NoTraverse: mt.noTraverse, NoTraverse: mt.noTraverse,
Callback: mt, Callback: mt,
DstIncludeAll: filter.Active.Opt.DeleteExcluded, DstIncludeAll: fi.Opt.DeleteExcluded,
} }
mt.processError(m.Run(ctx)) mt.processError(m.Run(ctx))
@ -260,6 +261,7 @@ func TestMarchNoTraverse(t *testing.T) {
cancel: cancel, cancel: cancel,
noTraverse: true, noTraverse: true,
} }
fi := filter.GetConfig(ctx)
m := &March{ m := &March{
Ctx: ctx, Ctx: ctx,
Fdst: r.Fremote, Fdst: r.Fremote,
@ -267,7 +269,7 @@ func TestMarchNoTraverse(t *testing.T) {
Dir: "", Dir: "",
NoTraverse: mt.noTraverse, NoTraverse: mt.noTraverse,
Callback: mt, Callback: mt,
DstIncludeAll: filter.Active.Opt.DeleteExcluded, DstIncludeAll: fi.Opt.DeleteExcluded,
} }
mt.processError(m.Run(ctx)) mt.processError(m.Run(ctx))

View File

@ -18,9 +18,11 @@ func TestListDirSorted(t *testing.T) {
r := fstest.NewRun(t) r := fstest.NewRun(t)
defer r.Finalise() defer r.Finalise()
filter.Active.Opt.MaxSize = 10 ctx := context.Background()
fi := filter.GetConfig(ctx)
fi.Opt.MaxSize = 10
defer func() { defer func() {
filter.Active.Opt.MaxSize = -1 fi.Opt.MaxSize = -1
}() }()
files := []fstest.Item{ files := []fstest.Item{
@ -79,7 +81,7 @@ func TestListDirSorted(t *testing.T) {
assert.Equal(t, "sub dir/sub sub dir/", str(1)) assert.Equal(t, "sub dir/sub sub dir/", str(1))
// testing ignore file // testing ignore file
filter.Active.Opt.ExcludeFile = ".ignore" fi.Opt.ExcludeFile = ".ignore"
items, err = list.DirSorted(context.Background(), r.Fremote, false, "sub dir") items, err = list.DirSorted(context.Background(), r.Fremote, false, "sub dir")
require.NoError(t, err) require.NoError(t, err)
@ -96,7 +98,7 @@ func TestListDirSorted(t *testing.T) {
assert.Equal(t, "sub dir/ignore dir/.ignore", str(0)) assert.Equal(t, "sub dir/ignore dir/.ignore", str(0))
assert.Equal(t, "sub dir/ignore dir/should be ignored", str(1)) assert.Equal(t, "sub dir/ignore dir/should be ignored", str(1))
filter.Active.Opt.ExcludeFile = "" fi.Opt.ExcludeFile = ""
items, err = list.DirSorted(context.Background(), r.Fremote, false, "sub dir/ignore dir") items, err = list.DirSorted(context.Background(), r.Fremote, false, "sub dir/ignore dir")
require.NoError(t, err) require.NoError(t, err)
require.Len(t, items, 2) require.Len(t, items, 2)

View File

@ -120,12 +120,8 @@ func TestLsWithFilesFrom(t *testing.T) {
require.NoError(t, f.AddFile("potato2")) require.NoError(t, f.AddFile("potato2"))
require.NoError(t, f.AddFile("notfound")) require.NoError(t, f.AddFile("notfound"))
// Monkey patch the active filter // Change the active filter
oldFilter := filter.Active ctx = filter.ReplaceConfig(ctx, f)
filter.Active = f
defer func() {
filter.Active = oldFilter
}()
var buf bytes.Buffer var buf bytes.Buffer
err = operations.List(ctx, r.Fremote, &buf) err = operations.List(ctx, r.Fremote, &buf)
@ -321,6 +317,7 @@ func TestCount(t *testing.T) {
func TestDelete(t *testing.T) { func TestDelete(t *testing.T) {
ctx := context.Background() ctx := context.Background()
fi := filter.GetConfig(ctx)
r := fstest.NewRun(t) r := fstest.NewRun(t)
defer r.Finalise() defer r.Finalise()
file1 := r.WriteObject(ctx, "small", "1234567890", t2) // 10 bytes file1 := r.WriteObject(ctx, "small", "1234567890", t2) // 10 bytes
@ -328,9 +325,9 @@ func TestDelete(t *testing.T) {
file3 := r.WriteObject(ctx, "large", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 100 bytes file3 := r.WriteObject(ctx, "large", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 100 bytes
fstest.CheckItems(t, r.Fremote, file1, file2, file3) fstest.CheckItems(t, r.Fremote, file1, file2, file3)
filter.Active.Opt.MaxSize = 60 fi.Opt.MaxSize = 60
defer func() { defer func() {
filter.Active.Opt.MaxSize = -1 fi.Opt.MaxSize = -1
}() }()
err := operations.Delete(ctx, r.Fremote) err := operations.Delete(ctx, r.Fremote)

View File

@ -31,6 +31,7 @@ type syncCopyMove struct {
dir string dir string
// internal state // internal state
ci *fs.ConfigInfo // global config ci *fs.ConfigInfo // global config
fi *filter.Filter // filter config
ctx context.Context // internal context for controlling go-routines ctx context.Context // internal context for controlling go-routines
cancel func() // cancel the context cancel func() // cancel the context
inCtx context.Context // internal context for controlling march inCtx context.Context // internal context for controlling march
@ -99,8 +100,10 @@ func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete
return nil, fserrors.FatalError(fs.ErrorOverlapping) return nil, fserrors.FatalError(fs.ErrorOverlapping)
} }
ci := fs.GetConfig(ctx) ci := fs.GetConfig(ctx)
fi := filter.GetConfig(ctx)
s := &syncCopyMove{ s := &syncCopyMove{
ci: ci, ci: ci,
fi: fi,
fdst: fdst, fdst: fdst,
fsrc: fsrc, fsrc: fsrc,
deleteMode: deleteMode, deleteMode: deleteMode,
@ -828,7 +831,7 @@ func (s *syncCopyMove) run() error {
Dir: s.dir, Dir: s.dir,
NoTraverse: s.noTraverse, NoTraverse: s.noTraverse,
Callback: s, Callback: s,
DstIncludeAll: filter.Active.Opt.DeleteExcluded, DstIncludeAll: s.fi.Opt.DeleteExcluded,
NoCheckDest: s.noCheckDest, NoCheckDest: s.noCheckDest,
NoUnicodeNormalization: s.noUnicodeNormalization, NoUnicodeNormalization: s.noUnicodeNormalization,
} }
@ -1087,13 +1090,14 @@ func moveDir(ctx context.Context, fdst, fsrc fs.Fs, deleteEmptySrcDirs bool, cop
// MoveDir moves fsrc into fdst // MoveDir moves fsrc into fdst
func MoveDir(ctx context.Context, fdst, fsrc fs.Fs, deleteEmptySrcDirs bool, copyEmptySrcDirs bool) error { func MoveDir(ctx context.Context, fdst, fsrc fs.Fs, deleteEmptySrcDirs bool, copyEmptySrcDirs bool) error {
fi := filter.GetConfig(ctx)
if operations.Same(fdst, fsrc) { if operations.Same(fdst, fsrc) {
fs.Errorf(fdst, "Nothing to do as source and destination are the same") fs.Errorf(fdst, "Nothing to do as source and destination are the same")
return nil return nil
} }
// First attempt to use DirMover if exists, same Fs and no filters are active // First attempt to use DirMover if exists, same Fs and no filters are active
if fdstDirMove := fdst.Features().DirMove; fdstDirMove != nil && operations.SameConfig(fsrc, fdst) && filter.Active.InActive() { if fdstDirMove := fdst.Features().DirMove; fdstDirMove != nil && operations.SameConfig(fsrc, fdst) && fi.InActive() {
if operations.SkipDestructive(ctx, fdst, "server-side directory move") { if operations.SkipDestructive(ctx, fdst, "server-side directory move") {
return nil return nil
} }

View File

@ -177,13 +177,12 @@ func testCopyWithFilesFrom(t *testing.T, noTraverse bool) {
require.NoError(t, f.AddFile("potato2")) require.NoError(t, f.AddFile("potato2"))
require.NoError(t, f.AddFile("notfound")) require.NoError(t, f.AddFile("notfound"))
// Monkey patch the active filter // Change the active filter
oldFilter := filter.Active ctx = filter.ReplaceConfig(ctx, f)
oldNoTraverse := ci.NoTraverse oldNoTraverse := ci.NoTraverse
filter.Active = f
ci.NoTraverse = noTraverse ci.NoTraverse = noTraverse
unpatch := func() { unpatch := func() {
filter.Active = oldFilter
ci.NoTraverse = oldNoTraverse ci.NoTraverse = oldNoTraverse
} }
defer unpatch() defer unpatch()
@ -967,9 +966,10 @@ func TestSyncWithExclude(t *testing.T) {
fstest.CheckItems(t, r.Fremote, file1, file2) fstest.CheckItems(t, r.Fremote, file1, file2)
fstest.CheckItems(t, r.Flocal, file1, file2, file3) fstest.CheckItems(t, r.Flocal, file1, file2, file3)
filter.Active.Opt.MaxSize = 40 fi := filter.GetConfig(ctx)
fi.Opt.MaxSize = 40
defer func() { defer func() {
filter.Active.Opt.MaxSize = -1 fi.Opt.MaxSize = -1
}() }()
accounting.GlobalStats().ResetCounters() accounting.GlobalStats().ResetCounters()
@ -996,11 +996,12 @@ func TestSyncWithExcludeAndDeleteExcluded(t *testing.T) {
fstest.CheckItems(t, r.Fremote, file1, file2, file3) fstest.CheckItems(t, r.Fremote, file1, file2, file3)
fstest.CheckItems(t, r.Flocal, file1, file2, file3) fstest.CheckItems(t, r.Flocal, file1, file2, file3)
filter.Active.Opt.MaxSize = 40 fi := filter.GetConfig(ctx)
filter.Active.Opt.DeleteExcluded = true fi.Opt.MaxSize = 40
fi.Opt.DeleteExcluded = true
defer func() { defer func() {
filter.Active.Opt.MaxSize = -1 fi.Opt.MaxSize = -1
filter.Active.Opt.DeleteExcluded = false fi.Opt.DeleteExcluded = false
}() }()
accounting.GlobalStats().ResetCounters() accounting.GlobalStats().ResetCounters()
@ -1399,12 +1400,14 @@ func TestServerSideMove(t *testing.T) {
// Test a server-side move if possible, or the backup path if not // Test a server-side move if possible, or the backup path if not
func TestServerSideMoveWithFilter(t *testing.T) { func TestServerSideMoveWithFilter(t *testing.T) {
ctx := context.Background()
r := fstest.NewRun(t) r := fstest.NewRun(t)
defer r.Finalise() defer r.Finalise()
filter.Active.Opt.MinSize = 40 fi := filter.GetConfig(ctx)
fi.Opt.MinSize = 40
defer func() { defer func() {
filter.Active.Opt.MinSize = -1 fi.Opt.MinSize = -1
}() }()
testServerSideMove(t, r, true, false) testServerSideMove(t, r, true, false)
@ -1439,9 +1442,10 @@ func TestServerSideMoveOverlap(t *testing.T) {
assert.EqualError(t, err, fs.ErrorOverlapping.Error()) assert.EqualError(t, err, fs.ErrorOverlapping.Error())
// Now try with a filter which should also fail with ErrorCantMoveOverlapping // Now try with a filter which should also fail with ErrorCantMoveOverlapping
filter.Active.Opt.MinSize = 40 fi := filter.GetConfig(ctx)
fi.Opt.MinSize = 40
defer func() { defer func() {
filter.Active.Opt.MinSize = -1 fi.Opt.MinSize = -1
}() }()
err = MoveDir(ctx, FremoteMove, r.Fremote, false, false) err = MoveDir(ctx, FremoteMove, r.Fremote, false, false)
assert.EqualError(t, err, fs.ErrorOverlapping.Error()) assert.EqualError(t, err, fs.ErrorOverlapping.Error())
@ -1686,11 +1690,8 @@ func testSyncBackupDir(t *testing.T, backupDir string, suffix string, suffixKeep
flt, err := filter.NewFilter(nil) flt, err := filter.NewFilter(nil)
require.NoError(t, err) require.NoError(t, err)
require.NoError(t, flt.AddRule("- *"+suffix)) require.NoError(t, flt.AddRule("- *"+suffix))
oldFlt := filter.Active // Change the active filter
filter.Active = flt ctx = filter.ReplaceConfig(ctx, flt)
defer func() {
filter.Active = oldFlt
}()
} }
ci.Suffix = suffix ci.Suffix = suffix
ci.SuffixKeepExtension = suffixKeepExtension ci.SuffixKeepExtension = suffixKeepExtension

View File

@ -60,8 +60,9 @@ type Func func(path string, entries fs.DirEntries, err error) error
// NB (f, path) to be replaced by fs.Dir at some point // NB (f, path) to be replaced by fs.Dir at some point
func Walk(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel int, fn Func) error { func Walk(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel int, fn Func) error {
ci := fs.GetConfig(ctx) ci := fs.GetConfig(ctx)
if ci.NoTraverse && filter.Active.HaveFilesFrom() { fi := filter.GetConfig(ctx)
return walkR(ctx, f, path, includeAll, maxLevel, fn, filter.Active.MakeListR(ctx, f.NewObject)) if ci.NoTraverse && fi.HaveFilesFrom() {
return walkR(ctx, f, path, includeAll, maxLevel, fn, fi.MakeListR(ctx, f.NewObject))
} }
// FIXME should this just be maxLevel < 0 - why the maxLevel > 1 // FIXME should this just be maxLevel < 0 - why the maxLevel > 1
if (maxLevel < 0 || maxLevel > 1) && ci.UseListR && f.Features().ListR != nil { if (maxLevel < 0 || maxLevel > 1) && ci.UseListR && f.Features().ListR != nil {
@ -139,15 +140,16 @@ func (l ListType) Filter(in *fs.DirEntries) {
// //
// NB (f, path) to be replaced by fs.Dir at some point // NB (f, path) to be replaced by fs.Dir at some point
func ListR(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel int, listType ListType, fn fs.ListRCallback) error { func ListR(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel int, listType ListType, fn fs.ListRCallback) error {
fi := filter.GetConfig(ctx)
// FIXME disable this with --no-fast-list ??? `--disable ListR` will do it... // FIXME disable this with --no-fast-list ??? `--disable ListR` will do it...
doListR := f.Features().ListR doListR := f.Features().ListR
// Can't use ListR if... // Can't use ListR if...
if doListR == nil || // ...no ListR if doListR == nil || // ...no ListR
filter.Active.HaveFilesFrom() || // ...using --files-from fi.HaveFilesFrom() || // ...using --files-from
maxLevel >= 0 || // ...using bounded recursion maxLevel >= 0 || // ...using bounded recursion
len(filter.Active.Opt.ExcludeFile) > 0 || // ...using --exclude-file len(fi.Opt.ExcludeFile) > 0 || // ...using --exclude-file
filter.Active.UsesDirectoryFilters() { // ...using any directory filters fi.UsesDirectoryFilters() { // ...using any directory filters
return listRwalk(ctx, f, path, includeAll, maxLevel, listType, fn) return listRwalk(ctx, f, path, includeAll, maxLevel, listType, fn)
} }
return listR(ctx, f, path, includeAll, listType, fn, doListR, listType.Dirs() && f.Features().BucketBased) return listR(ctx, f, path, includeAll, listType, fn, doListR, listType.Dirs() && f.Features().BucketBased)
@ -275,9 +277,10 @@ func (dm *dirMap) sendEntries(fn fs.ListRCallback) (err error) {
// listR walks the file tree using ListR // listR walks the file tree using ListR
func listR(ctx context.Context, f fs.Fs, path string, includeAll bool, listType ListType, fn fs.ListRCallback, doListR fs.ListRFn, synthesizeDirs bool) error { func listR(ctx context.Context, f fs.Fs, path string, includeAll bool, listType ListType, fn fs.ListRCallback, doListR fs.ListRFn, synthesizeDirs bool) error {
includeDirectory := filter.Active.IncludeDirectory(ctx, f) fi := filter.GetConfig(ctx)
includeDirectory := fi.IncludeDirectory(ctx, f)
if !includeAll { if !includeAll {
includeAll = filter.Active.InActive() includeAll = fi.InActive()
} }
var dm *dirMap var dm *dirMap
if synthesizeDirs { if synthesizeDirs {
@ -298,7 +301,7 @@ func listR(ctx context.Context, f fs.Fs, path string, includeAll bool, listType
var include bool var include bool
switch x := entry.(type) { switch x := entry.(type) {
case fs.Object: case fs.Object:
include = filter.Active.IncludeObject(ctx, x) include = fi.IncludeObject(ctx, x)
case fs.Directory: case fs.Directory:
include, err = includeDirectory(x.Remote()) include, err = includeDirectory(x.Remote())
if err != nil { if err != nil {
@ -448,11 +451,12 @@ func walk(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel i
} }
func walkRDirTree(ctx context.Context, f fs.Fs, startPath string, includeAll bool, maxLevel int, listR fs.ListRFn) (dirtree.DirTree, error) { func walkRDirTree(ctx context.Context, f fs.Fs, startPath string, includeAll bool, maxLevel int, listR fs.ListRFn) (dirtree.DirTree, error) {
fi := filter.GetConfig(ctx)
dirs := dirtree.New() dirs := dirtree.New()
// Entries can come in arbitrary order. We use toPrune to keep // Entries can come in arbitrary order. We use toPrune to keep
// all directories to exclude later. // all directories to exclude later.
toPrune := make(map[string]bool) toPrune := make(map[string]bool)
includeDirectory := filter.Active.IncludeDirectory(ctx, f) includeDirectory := fi.IncludeDirectory(ctx, f)
var mu sync.Mutex var mu sync.Mutex
err := listR(ctx, startPath, func(entries fs.DirEntries) error { err := listR(ctx, startPath, func(entries fs.DirEntries) error {
mu.Lock() mu.Lock()
@ -462,7 +466,7 @@ func walkRDirTree(ctx context.Context, f fs.Fs, startPath string, includeAll boo
switch x := entry.(type) { switch x := entry.(type) {
case fs.Object: case fs.Object:
// Make sure we don't delete excluded files if not required // Make sure we don't delete excluded files if not required
if includeAll || filter.Active.IncludeObject(ctx, x) { if includeAll || fi.IncludeObject(ctx, x) {
if maxLevel < 0 || slashes <= maxLevel-1 { if maxLevel < 0 || slashes <= maxLevel-1 {
dirs.Add(x) dirs.Add(x)
} else { } else {
@ -477,9 +481,9 @@ func walkRDirTree(ctx context.Context, f fs.Fs, startPath string, includeAll boo
fs.Debugf(x, "Excluded from sync (and deletion)") fs.Debugf(x, "Excluded from sync (and deletion)")
} }
// Check if we need to prune a directory later. // Check if we need to prune a directory later.
if !includeAll && len(filter.Active.Opt.ExcludeFile) > 0 { if !includeAll && len(fi.Opt.ExcludeFile) > 0 {
basename := path.Base(x.Remote()) basename := path.Base(x.Remote())
if basename == filter.Active.Opt.ExcludeFile { if basename == fi.Opt.ExcludeFile {
excludeDir := parentDir(x.Remote()) excludeDir := parentDir(x.Remote())
toPrune[excludeDir] = true toPrune[excludeDir] = true
fs.Debugf(basename, "Excluded from sync (and deletion) based on exclude file") fs.Debugf(basename, "Excluded from sync (and deletion) based on exclude file")
@ -556,12 +560,13 @@ func walkNDirTree(ctx context.Context, f fs.Fs, path string, includeAll bool, ma
// NB (f, path) to be replaced by fs.Dir at some point // NB (f, path) to be replaced by fs.Dir at some point
func NewDirTree(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel int) (dirtree.DirTree, error) { func NewDirTree(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel int) (dirtree.DirTree, error) {
ci := fs.GetConfig(ctx) ci := fs.GetConfig(ctx)
fi := filter.GetConfig(ctx)
// if --no-traverse and --files-from build DirTree just from files // if --no-traverse and --files-from build DirTree just from files
if ci.NoTraverse && filter.Active.HaveFilesFrom() { if ci.NoTraverse && fi.HaveFilesFrom() {
return walkRDirTree(ctx, f, path, includeAll, maxLevel, filter.Active.MakeListR(ctx, f.NewObject)) return walkRDirTree(ctx, f, path, includeAll, maxLevel, fi.MakeListR(ctx, f.NewObject))
} }
// if have ListR; and recursing; and not using --files-from; then build a DirTree with ListR // if have ListR; and recursing; and not using --files-from; then build a DirTree with ListR
if ListR := f.Features().ListR; (maxLevel < 0 || maxLevel > 1) && ListR != nil && !filter.Active.HaveFilesFrom() { if ListR := f.Features().ListR; (maxLevel < 0 || maxLevel > 1) && ListR != nil && !fi.HaveFilesFrom() {
return walkRDirTree(ctx, f, path, includeAll, maxLevel, ListR) return walkRDirTree(ctx, f, path, includeAll, maxLevel, ListR)
} }
// otherwise just use List // otherwise just use List

View File

@ -585,6 +585,8 @@ a/
} }
func TestWalkRDirTreeExclude(t *testing.T) { func TestWalkRDirTreeExclude(t *testing.T) {
ctx := context.Background()
fi := filter.GetConfig(ctx)
for _, test := range []struct { for _, test := range []struct {
entries fs.DirEntries entries fs.DirEntries
want string want string
@ -648,13 +650,13 @@ b/c/d/
e e
`, nil, "", -1, "ign", true}, `, nil, "", -1, "ign", true},
} { } {
filter.Active.Opt.ExcludeFile = test.excludeFile fi.Opt.ExcludeFile = test.excludeFile
r, err := walkRDirTree(context.Background(), nil, test.root, test.includeAll, test.level, makeListRCallback(test.entries, test.err)) r, err := walkRDirTree(context.Background(), nil, test.root, test.includeAll, test.level, makeListRCallback(test.entries, test.err))
assert.Equal(t, test.err, err, fmt.Sprintf("%+v", test)) assert.Equal(t, test.err, err, fmt.Sprintf("%+v", test))
assert.Equal(t, test.want, r.String(), fmt.Sprintf("%+v", test)) assert.Equal(t, test.want, r.String(), fmt.Sprintf("%+v", test))
} }
// Set to default value, to avoid side effects // Set to default value, to avoid side effects
filter.Active.Opt.ExcludeFile = "" fi.Opt.ExcludeFile = ""
} }
func TestListType(t *testing.T) { func TestListType(t *testing.T) {
@ -701,6 +703,7 @@ func TestListType(t *testing.T) {
} }
func TestListR(t *testing.T) { func TestListR(t *testing.T) {
ctx := context.Background()
objects := fs.DirEntries{ objects := fs.DirEntries{
mockobject.Object("a"), mockobject.Object("a"),
mockobject.Object("b"), mockobject.Object("b"),
@ -709,7 +712,7 @@ func TestListR(t *testing.T) {
mockobject.Object("dir/b"), mockobject.Object("dir/b"),
mockobject.Object("dir/c"), mockobject.Object("dir/c"),
} }
f := mockfs.NewFs(context.Background(), "mock", "/") f := mockfs.NewFs(ctx, "mock", "/")
var got []string var got []string
clearCallback := func() { clearCallback := func() {
got = nil got = nil
@ -730,57 +733,53 @@ func TestListR(t *testing.T) {
return callback(os) return callback(os)
} }
// Setup filter fi, err := filter.NewFilter(nil)
oldFilter := filter.Active
defer func() {
filter.Active = oldFilter
}()
var err error
filter.Active, err = filter.NewFilter(nil)
require.NoError(t, err) require.NoError(t, err)
require.NoError(t, filter.Active.AddRule("+ b")) require.NoError(t, fi.AddRule("+ b"))
require.NoError(t, filter.Active.AddRule("- *")) require.NoError(t, fi.AddRule("- *"))
// Change the active filter
ctx = filter.ReplaceConfig(ctx, fi)
// Base case // Base case
clearCallback() clearCallback()
err = listR(context.Background(), f, "", true, ListAll, callback, doListR, false) err = listR(ctx, f, "", true, ListAll, callback, doListR, false)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, []string{"a", "b", "dir", "dir/a", "dir/b", "dir/c"}, got) require.Equal(t, []string{"a", "b", "dir", "dir/a", "dir/b", "dir/c"}, got)
// Base case - with Objects // Base case - with Objects
clearCallback() clearCallback()
err = listR(context.Background(), f, "", true, ListObjects, callback, doListR, false) err = listR(ctx, f, "", true, ListObjects, callback, doListR, false)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, []string{"a", "b", "dir/a", "dir/b", "dir/c"}, got) require.Equal(t, []string{"a", "b", "dir/a", "dir/b", "dir/c"}, got)
// Base case - with Dirs // Base case - with Dirs
clearCallback() clearCallback()
err = listR(context.Background(), f, "", true, ListDirs, callback, doListR, false) err = listR(ctx, f, "", true, ListDirs, callback, doListR, false)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, []string{"dir"}, got) require.Equal(t, []string{"dir"}, got)
// With filter // With filter
clearCallback() clearCallback()
err = listR(context.Background(), f, "", false, ListAll, callback, doListR, false) err = listR(ctx, f, "", false, ListAll, callback, doListR, false)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, []string{"b", "dir", "dir/b"}, got) require.Equal(t, []string{"b", "dir", "dir/b"}, got)
// With filter - with Objects // With filter - with Objects
clearCallback() clearCallback()
err = listR(context.Background(), f, "", false, ListObjects, callback, doListR, false) err = listR(ctx, f, "", false, ListObjects, callback, doListR, false)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, []string{"b", "dir/b"}, got) require.Equal(t, []string{"b", "dir/b"}, got)
// With filter - with Dir // With filter - with Dir
clearCallback() clearCallback()
err = listR(context.Background(), f, "", false, ListDirs, callback, doListR, false) err = listR(ctx, f, "", false, ListDirs, callback, doListR, false)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, []string{"dir"}, got) require.Equal(t, []string{"dir"}, got)
// With filter and subdir // With filter and subdir
clearCallback() clearCallback()
err = listR(context.Background(), f, "dir", false, ListAll, callback, doListR, false) err = listR(ctx, f, "dir", false, ListAll, callback, doListR, false)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, []string{"dir/b"}, got) require.Equal(t, []string{"dir/b"}, got)
@ -796,31 +795,31 @@ func TestListR(t *testing.T) {
// Base case // Base case
clearCallback() clearCallback()
err = listR(context.Background(), f, "", true, ListAll, callback, doListR, true) err = listR(ctx, f, "", true, ListAll, callback, doListR, true)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, []string{"a", "b", "dir/a", "dir/b", "dir/subdir/c", "dir/subdir", "dir"}, got) require.Equal(t, []string{"a", "b", "dir/a", "dir/b", "dir/subdir/c", "dir/subdir", "dir"}, got)
// With filter // With filter
clearCallback() clearCallback()
err = listR(context.Background(), f, "", false, ListAll, callback, doListR, true) err = listR(ctx, f, "", false, ListAll, callback, doListR, true)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, []string{"b", "dir/b", "dir/subdir", "dir"}, got) require.Equal(t, []string{"b", "dir/b", "dir/subdir", "dir"}, got)
// With filter and subdir // With filter and subdir
clearCallback() clearCallback()
err = listR(context.Background(), f, "dir", false, ListAll, callback, doListR, true) err = listR(ctx, f, "dir", false, ListAll, callback, doListR, true)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, []string{"dir/b", "dir/subdir"}, got) require.Equal(t, []string{"dir/b", "dir/subdir"}, got)
// With filter and subdir - with Objects // With filter and subdir - with Objects
clearCallback() clearCallback()
err = listR(context.Background(), f, "dir", false, ListObjects, callback, doListR, true) err = listR(ctx, f, "dir", false, ListObjects, callback, doListR, true)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, []string{"dir/b"}, got) require.Equal(t, []string{"dir/b"}, got)
// With filter and subdir - with Dirs // With filter and subdir - with Dirs
clearCallback() clearCallback()
err = listR(context.Background(), f, "dir", false, ListDirs, callback, doListR, true) err = listR(ctx, f, "dir", false, ListDirs, callback, doListR, true)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, []string{"dir/subdir"}, got) require.Equal(t, []string{"dir/subdir"}, got)
} }