diff --git a/fs/filter/filter.go b/fs/filter/filter.go index 08078a2b4..16ee3d128 100644 --- a/fs/filter/filter.go +++ b/fs/filter/filter.go @@ -21,8 +21,9 @@ var Active = mustNewFilter(nil) // rule is one filter rule type rule struct { - Include bool - Regexp *regexp.Regexp + Include bool + Regexp *regexp.Regexp + boundedRecursion bool } // Match returns true if rule matches path @@ -46,13 +47,14 @@ type rules struct { } // add adds a rule if it doesn't exist already -func (rs *rules) add(Include bool, re *regexp.Regexp) { +func (rs *rules) add(Include bool, re *regexp.Regexp, boundedRecursion bool) { if rs.existing == nil { rs.existing = make(map[string]struct{}) } newRule := rule{ - Include: Include, - Regexp: re, + Include: Include, + Regexp: re, + boundedRecursion: boundedRecursion, } newRuleString := newRule.String() if _, ok := rs.existing[newRuleString]; ok { @@ -73,6 +75,23 @@ func (rs *rules) len() int { return len(rs.rules) } +// boundedRecursion returns true if the set of filters would only +// need bounded recursion to evaluate +func (rs *rules) boundedRecursion() bool { + var ( + excludeAll = false + boundedRecursion = true + ) + for _, rule := range rs.rules { + if rule.Include { + boundedRecursion = boundedRecursion && rule.boundedRecursion + } else if rule.Regexp.String() == `^.*$` { + excludeAll = true + } + } + return excludeAll && boundedRecursion +} + // FilesMap describes the map of files to transfer type FilesMap map[string]struct{} @@ -232,7 +251,8 @@ func (f *Filter) addDirGlobs(Include bool, glob string) error { if err != nil { return err } - f.dirRules.add(Include, dirRe) + boundedRecursion := globBoundedRecursion(dirGlob) + f.dirRules.add(Include, dirRe, boundedRecursion) } return nil } @@ -248,8 +268,9 @@ func (f *Filter) Add(Include bool, glob string) error { if err != nil { return err } + boundedRecursion := globBoundedRecursion(glob) if isFileRule { - f.fileRules.add(Include, re) + f.fileRules.add(Include, re, boundedRecursion) // If include rule work out what directories are needed to scan // if exclude rule, we can't rule anything out // Unless it is `*` which matches everything @@ -262,7 +283,7 @@ func (f *Filter) Add(Include bool, glob string) error { } } if isDirRule { - f.dirRules.add(Include, re) + f.dirRules.add(Include, re, boundedRecursion) } return nil } @@ -343,6 +364,12 @@ func (f *Filter) InActive() bool { len(f.Opt.ExcludeFile) == 0) } +// BoundedRecursion returns true if the filter can be evaluated with +// bounded recursion only. +func (f *Filter) BoundedRecursion() bool { + return f.fileRules.boundedRecursion() +} + // includeRemote returns whether this remote passes the filter rules. func (f *Filter) includeRemote(remote string) bool { for _, rule := range f.fileRules.rules { diff --git a/fs/filter/filter_test.go b/fs/filter/filter_test.go index 236392149..629165be0 100644 --- a/fs/filter/filter_test.go +++ b/fs/filter/filter_test.go @@ -25,6 +25,7 @@ func TestNewFilterDefault(t *testing.T) { assert.Len(t, f.dirRules.rules, 0) assert.Nil(t, f.files) assert.True(t, f.InActive()) + assert.False(t, f.BoundedRecursion()) } // testFile creates a temp file with the contents @@ -103,6 +104,38 @@ func TestNewFilterFull(t *testing.T) { } } assert.False(t, f.InActive()) + assert.False(t, f.BoundedRecursion()) +} + +func TestFilterBoundedRecursion(t *testing.T) { + for _, test := range []struct { + in string + want bool + }{ + {"", false}, + {"- /**", true}, + {"+ *.jpg", false}, + {"+ *.jpg\n- /**", false}, + {"+ /*.jpg\n- /**", true}, + {"+ *.png\n+ /*.jpg\n- /**", false}, + {"+ /*.png\n+ /*.jpg\n- /**", true}, + {"- *.jpg\n- /**", true}, + {"+ /*.jpg\n- /**", true}, + {"+ /*dir/\n- /**", true}, + {"+ /*dir/\n", false}, + {"+ /*dir/**\n- /**", false}, + {"+ **/pics*/*.jpg\n- /**", false}, + } { + f, err := NewFilter(nil) + require.NoError(t, err) + for _, rule := range strings.Split(test.in, "\n") { + if rule != "" { + require.NoError(t, f.AddRule(rule)) + } + } + got := f.BoundedRecursion() + assert.Equal(t, test.want, got, test.in) + } } type includeTest struct { @@ -151,6 +184,7 @@ func TestNewFilterIncludeFiles(t *testing.T) { {"file3.jpg", 3, 0, false}, }) assert.False(t, f.InActive()) + assert.False(t, f.BoundedRecursion()) } func TestNewFilterIncludeFilesDirs(t *testing.T) { @@ -278,6 +312,7 @@ func TestNewFilterMinSize(t *testing.T) { {"potato/file2.jpg", 99, 0, false}, }) assert.False(t, f.InActive()) + assert.False(t, f.BoundedRecursion()) } func TestNewFilterMaxSize(t *testing.T) { diff --git a/fs/filter/glob.go b/fs/filter/glob.go index 96a48d3d4..9fff1ca73 100644 --- a/fs/filter/glob.go +++ b/fs/filter/glob.go @@ -167,3 +167,15 @@ func globToDirGlobs(glob string) (out []string) { return out } + +// globBoundedRecursion returns true if the glob only needs bounded +// recursion in the file tree to evaluate. +func globBoundedRecursion(glob string) bool { + if strings.Contains(glob, "**") { + return false + } + if strings.HasPrefix(glob, "/") { + return true + } + return false +} diff --git a/fs/filter/glob_test.go b/fs/filter/glob_test.go index 008d4bfd3..372fa63e3 100644 --- a/fs/filter/glob_test.go +++ b/fs/filter/glob_test.go @@ -108,3 +108,45 @@ func TestGlobToDirGlobs(t *testing.T) { assert.Equal(t, test.want, got, test.in) } } + +func TestGlobBoundedRecursion(t *testing.T) { + for _, test := range []struct { + in string + want bool + }{ + {`*`, false}, + {`/*`, true}, + {`/**`, false}, + {`*.jpg`, false}, + {`/*.jpg`, true}, + {`/a/*.jpg`, true}, + {`/a/b/*.jpg`, true}, + {`*/*/*.jpg`, false}, + {`a/b/`, false}, + {`a/b`, false}, + {`a/b/*.{png,gif}`, false}, + {`/a/{jpg,png,gif}/*.{jpg,true,gif}`, true}, + {`a/{a,a*b,a**c}/d/`, false}, + {`/a/{a,a*b,a/c,d}/d/`, true}, + {`**`, false}, + {`a**`, false}, + {`a**b`, false}, + {`a**b**c**d`, false}, + {`a**b/c**d`, false}, + {`/A/a**b/B/c**d/C/`, false}, + {`/var/spool/**/ncw`, false}, + {`var/spool/**/ncw/`, false}, + {"/file1.jpg", true}, + {"/file2.png", true}, + {"/*.jpg", true}, + {"/*.png", true}, + {"/potato", true}, + {"/sausage1", true}, + {"/sausage2*", true}, + {"/sausage3**", false}, + {"/a/*.jpg", true}, + } { + got := globBoundedRecursion(test.in) + assert.Equal(t, test.want, got, test.in) + } +}