From bd787e8f457705de9072335fda879f575da4f3cf Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Mon, 5 Sep 2022 16:19:50 +0100 Subject: [PATCH] filter: Fix incorrect filtering with UseFilter context flag and wrapping backends In this commit 8d1fff9a8237c64f local: obey file filters in listing to fix errors on excluded files We started using filters in the local backend so the user could short circuit troublesome files/directories at a low level. However this caused a number of integration tests to fail. This turned out to be in backends wrapping the local backend. For example the combine backend test failed because it changes the paths passed to the local backend so they no longer match the paths in the current filter. To fix this, a new feature flag `FilterAware` was added and the UseFilter context flag is only passed to backends which support it. As the wrapping backends don't support the flag, this fixes the problems in the integration tests. In future the wrapping backends could modify the active filters to match the path modifications and then they could set the FilterAware flag. See #6376 --- backend/drive/drive.go | 1 + backend/drive/drive_internal_test.go | 3 +++ backend/local/local.go | 1 + backend/local/local_internal_test.go | 3 +++ fs/features.go | 2 ++ fs/march/march.go | 4 ++-- fs/walk/walk.go | 4 ++-- 7 files changed, 14 insertions(+), 4 deletions(-) diff --git a/backend/drive/drive.go b/backend/drive/drive.go index 30c165b61..e0b4abb61 100644 --- a/backend/drive/drive.go +++ b/backend/drive/drive.go @@ -1210,6 +1210,7 @@ func newFs(ctx context.Context, name, path string, m configmap.Mapper) (*Fs, err WriteMimeType: true, CanHaveEmptyDirectories: true, ServerSideAcrossConfigs: opt.ServerSideAcrossConfigs, + FilterAware: true, }).Fill(ctx, f) // Create a new authorized Drive client. diff --git a/backend/drive/drive_internal_test.go b/backend/drive/drive_internal_test.go index 22a84b0bb..c17da7062 100644 --- a/backend/drive/drive_internal_test.go +++ b/backend/drive/drive_internal_test.go @@ -518,6 +518,9 @@ func (f *Fs) InternalTestCopyID(t *testing.T) { // TestIntegration/FsMkdir/FsPutFiles/Internal/AgeQuery func (f *Fs) InternalTestAgeQuery(t *testing.T) { + // Check set up for filtering + assert.True(t, f.Features().FilterAware) + opt := &filter.Opt{} err := opt.MaxAge.Set("1h") assert.NoError(t, err) diff --git a/backend/local/local.go b/backend/local/local.go index 18763ace3..7d56d8012 100644 --- a/backend/local/local.go +++ b/backend/local/local.go @@ -300,6 +300,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e ReadMetadata: true, WriteMetadata: true, UserMetadata: xattrSupported, // can only R/W general purpose metadata if xattrs are supported + FilterAware: true, }).Fill(ctx, f) if opt.FollowSymlinks { f.lstat = os.Stat diff --git a/backend/local/local_internal_test.go b/backend/local/local_internal_test.go index 811831d91..7ecc9986b 100644 --- a/backend/local/local_internal_test.go +++ b/backend/local/local_internal_test.go @@ -378,6 +378,9 @@ func TestFilter(t *testing.T) { r.WriteFile("excluded", "excluded file", when) f := r.Flocal.(*Fs) + // Check set up for filtering + assert.True(t, f.Features().FilterAware) + // Add a filter ctx, fi := filter.AddConfig(ctx) require.NoError(t, fi.AddRule("+ included")) diff --git a/fs/features.go b/fs/features.go index 6fce02d60..b8bbffc21 100644 --- a/fs/features.go +++ b/fs/features.go @@ -29,6 +29,7 @@ type Features struct { ReadMetadata bool // can read metadata from objects WriteMetadata bool // can write metadata to objects UserMetadata bool // can read/write general purpose metadata + FilterAware bool // can make use of filters if provided for listing // Purge all files in the directory specified // @@ -320,6 +321,7 @@ func (ft *Features) Mask(ctx context.Context, f Fs) *Features { // ft.IsLocal = ft.IsLocal && mask.IsLocal Don't propagate IsLocal ft.SlowModTime = ft.SlowModTime && mask.SlowModTime ft.SlowHash = ft.SlowHash && mask.SlowHash + ft.FilterAware = ft.FilterAware && mask.FilterAware if mask.Purge == nil { ft.Purge = nil diff --git a/fs/march/march.go b/fs/march/march.go index db045156d..771f079f4 100644 --- a/fs/march/march.go +++ b/fs/march/march.go @@ -83,7 +83,7 @@ func (m *March) makeListDir(ctx context.Context, f fs.Fs, includeAll bool) listD if !(ci.UseListR && f.Features().ListR != nil) && // !--fast-list active and !(ci.NoTraverse && fi.HaveFilesFrom()) { // !(--files-from and --no-traverse) return func(dir string) (entries fs.DirEntries, err error) { - dirCtx := filter.SetUseFilter(m.Ctx, !includeAll) // make filter-aware backends constrain List + dirCtx := filter.SetUseFilter(m.Ctx, f.Features().FilterAware && !includeAll) // make filter-aware backends constrain List return list.DirSorted(dirCtx, f, includeAll, dir) } } @@ -100,7 +100,7 @@ func (m *March) makeListDir(ctx context.Context, f fs.Fs, includeAll bool) listD mu.Lock() defer mu.Unlock() if !started { - dirCtx := filter.SetUseFilter(m.Ctx, !includeAll) // make filter-aware backends constrain List + dirCtx := filter.SetUseFilter(m.Ctx, f.Features().FilterAware && !includeAll) // make filter-aware backends constrain List dirs, dirsErr = walk.NewDirTree(dirCtx, f, m.Dir, includeAll, ci.MaxDepth) started = true } diff --git a/fs/walk/walk.go b/fs/walk/walk.go index f63350a47..01251ea1b 100644 --- a/fs/walk/walk.go +++ b/fs/walk/walk.go @@ -64,7 +64,7 @@ type Func func(path string, entries fs.DirEntries, err error) error func Walk(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel int, fn Func) error { ci := fs.GetConfig(ctx) fi := filter.GetConfig(ctx) - ctx = filter.SetUseFilter(ctx, !includeAll) // make filter-aware backends constrain List + ctx = filter.SetUseFilter(ctx, f.Features().FilterAware && !includeAll) // make filter-aware backends constrain List if ci.NoTraverse && fi.HaveFilesFrom() { return walkR(ctx, f, path, includeAll, maxLevel, fn, fi.MakeListR(ctx, f.NewObject)) } @@ -158,7 +158,7 @@ func ListR(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel fi.UsesDirectoryFilters() { // ...using any directory filters return listRwalk(ctx, f, path, includeAll, maxLevel, listType, fn) } - ctx = filter.SetUseFilter(ctx, !includeAll) // make filter-aware backends constrain List + ctx = filter.SetUseFilter(ctx, f.Features().FilterAware && !includeAll) // make filter-aware backends constrain List return listR(ctx, f, path, includeAll, listType, fn, doListR, listType.Dirs() && f.Features().BucketBased) }