filter: Fix incorrect filtering with UseFilter context flag and wrapping backends

In this commit

8d1fff9a82 local: obey file filters in listing to fix errors on excluded files

We started using filters in the local backend so the user could short
circuit troublesome files/directories at a low level.

However this caused a number of integration tests to fail. This turned
out to be in backends wrapping the local backend. For example the
combine backend test failed because it changes the paths passed to the
local backend so they no longer match the paths in the current filter.

To fix this, a new feature flag `FilterAware` was added and the
UseFilter context flag is only passed to backends which support it. As
the wrapping backends don't support the flag, this fixes the problems
in the integration tests.

In future the wrapping backends could modify the active filters to
match the path modifications and then they could set the FilterAware
flag.

See #6376
This commit is contained in:
Nick Craig-Wood 2022-09-05 16:19:50 +01:00
parent 3cb7734eac
commit bd787e8f45
7 changed files with 14 additions and 4 deletions

View File

@ -1210,6 +1210,7 @@ func newFs(ctx context.Context, name, path string, m configmap.Mapper) (*Fs, err
WriteMimeType: true, WriteMimeType: true,
CanHaveEmptyDirectories: true, CanHaveEmptyDirectories: true,
ServerSideAcrossConfigs: opt.ServerSideAcrossConfigs, ServerSideAcrossConfigs: opt.ServerSideAcrossConfigs,
FilterAware: true,
}).Fill(ctx, f) }).Fill(ctx, f)
// Create a new authorized Drive client. // Create a new authorized Drive client.

View File

@ -518,6 +518,9 @@ func (f *Fs) InternalTestCopyID(t *testing.T) {
// TestIntegration/FsMkdir/FsPutFiles/Internal/AgeQuery // TestIntegration/FsMkdir/FsPutFiles/Internal/AgeQuery
func (f *Fs) InternalTestAgeQuery(t *testing.T) { func (f *Fs) InternalTestAgeQuery(t *testing.T) {
// Check set up for filtering
assert.True(t, f.Features().FilterAware)
opt := &filter.Opt{} opt := &filter.Opt{}
err := opt.MaxAge.Set("1h") err := opt.MaxAge.Set("1h")
assert.NoError(t, err) assert.NoError(t, err)

View File

@ -300,6 +300,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
ReadMetadata: true, ReadMetadata: true,
WriteMetadata: true, WriteMetadata: true,
UserMetadata: xattrSupported, // can only R/W general purpose metadata if xattrs are supported UserMetadata: xattrSupported, // can only R/W general purpose metadata if xattrs are supported
FilterAware: true,
}).Fill(ctx, f) }).Fill(ctx, f)
if opt.FollowSymlinks { if opt.FollowSymlinks {
f.lstat = os.Stat f.lstat = os.Stat

View File

@ -378,6 +378,9 @@ func TestFilter(t *testing.T) {
r.WriteFile("excluded", "excluded file", when) r.WriteFile("excluded", "excluded file", when)
f := r.Flocal.(*Fs) f := r.Flocal.(*Fs)
// Check set up for filtering
assert.True(t, f.Features().FilterAware)
// Add a filter // Add a filter
ctx, fi := filter.AddConfig(ctx) ctx, fi := filter.AddConfig(ctx)
require.NoError(t, fi.AddRule("+ included")) require.NoError(t, fi.AddRule("+ included"))

View File

@ -29,6 +29,7 @@ type Features struct {
ReadMetadata bool // can read metadata from objects ReadMetadata bool // can read metadata from objects
WriteMetadata bool // can write metadata to objects WriteMetadata bool // can write metadata to objects
UserMetadata bool // can read/write general purpose metadata UserMetadata bool // can read/write general purpose metadata
FilterAware bool // can make use of filters if provided for listing
// Purge all files in the directory specified // Purge all files in the directory specified
// //
@ -320,6 +321,7 @@ func (ft *Features) Mask(ctx context.Context, f Fs) *Features {
// ft.IsLocal = ft.IsLocal && mask.IsLocal Don't propagate IsLocal // ft.IsLocal = ft.IsLocal && mask.IsLocal Don't propagate IsLocal
ft.SlowModTime = ft.SlowModTime && mask.SlowModTime ft.SlowModTime = ft.SlowModTime && mask.SlowModTime
ft.SlowHash = ft.SlowHash && mask.SlowHash ft.SlowHash = ft.SlowHash && mask.SlowHash
ft.FilterAware = ft.FilterAware && mask.FilterAware
if mask.Purge == nil { if mask.Purge == nil {
ft.Purge = nil ft.Purge = nil

View File

@ -83,7 +83,7 @@ func (m *March) makeListDir(ctx context.Context, f fs.Fs, includeAll bool) listD
if !(ci.UseListR && f.Features().ListR != nil) && // !--fast-list active and if !(ci.UseListR && f.Features().ListR != nil) && // !--fast-list active and
!(ci.NoTraverse && fi.HaveFilesFrom()) { // !(--files-from and --no-traverse) !(ci.NoTraverse && fi.HaveFilesFrom()) { // !(--files-from and --no-traverse)
return func(dir string) (entries fs.DirEntries, err error) { return func(dir string) (entries fs.DirEntries, err error) {
dirCtx := filter.SetUseFilter(m.Ctx, !includeAll) // make filter-aware backends constrain List dirCtx := filter.SetUseFilter(m.Ctx, f.Features().FilterAware && !includeAll) // make filter-aware backends constrain List
return list.DirSorted(dirCtx, f, includeAll, dir) return list.DirSorted(dirCtx, f, includeAll, dir)
} }
} }
@ -100,7 +100,7 @@ func (m *March) makeListDir(ctx context.Context, f fs.Fs, includeAll bool) listD
mu.Lock() mu.Lock()
defer mu.Unlock() defer mu.Unlock()
if !started { if !started {
dirCtx := filter.SetUseFilter(m.Ctx, !includeAll) // make filter-aware backends constrain List dirCtx := filter.SetUseFilter(m.Ctx, f.Features().FilterAware && !includeAll) // make filter-aware backends constrain List
dirs, dirsErr = walk.NewDirTree(dirCtx, f, m.Dir, includeAll, ci.MaxDepth) dirs, dirsErr = walk.NewDirTree(dirCtx, f, m.Dir, includeAll, ci.MaxDepth)
started = true started = true
} }

View File

@ -64,7 +64,7 @@ type Func func(path string, entries fs.DirEntries, err error) error
func Walk(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel int, fn Func) error { func Walk(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel int, fn Func) error {
ci := fs.GetConfig(ctx) ci := fs.GetConfig(ctx)
fi := filter.GetConfig(ctx) fi := filter.GetConfig(ctx)
ctx = filter.SetUseFilter(ctx, !includeAll) // make filter-aware backends constrain List ctx = filter.SetUseFilter(ctx, f.Features().FilterAware && !includeAll) // make filter-aware backends constrain List
if ci.NoTraverse && fi.HaveFilesFrom() { if ci.NoTraverse && fi.HaveFilesFrom() {
return walkR(ctx, f, path, includeAll, maxLevel, fn, fi.MakeListR(ctx, f.NewObject)) return walkR(ctx, f, path, includeAll, maxLevel, fn, fi.MakeListR(ctx, f.NewObject))
} }
@ -158,7 +158,7 @@ func ListR(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel
fi.UsesDirectoryFilters() { // ...using any directory filters fi.UsesDirectoryFilters() { // ...using any directory filters
return listRwalk(ctx, f, path, includeAll, maxLevel, listType, fn) return listRwalk(ctx, f, path, includeAll, maxLevel, listType, fn)
} }
ctx = filter.SetUseFilter(ctx, !includeAll) // make filter-aware backends constrain List ctx = filter.SetUseFilter(ctx, f.Features().FilterAware && !includeAll) // make filter-aware backends constrain List
return listR(ctx, f, path, includeAll, listType, fn, doListR, listType.Dirs() && f.Features().BucketBased) return listR(ctx, f, path, includeAll, listType, fn, doListR, listType.Dirs() && f.Features().BucketBased)
} }