support exclude file in --fast-list mode

This commit is contained in:
Iakov Davydov 2017-11-09 10:28:36 +01:00 committed by Nick Craig-Wood
parent 557dd8f031
commit 538246f6c3
5 changed files with 83 additions and 23 deletions

View File

@ -406,22 +406,51 @@ func (f *Filter) ListContainsExcludeFile(entries DirEntries) bool {
return false
}
// IncludeDirectory returns whether this directory should be included
// in the sync or not.
func (f *Filter) IncludeDirectory(remote string) bool {
remote = strings.Trim(remote, "/")
// filesFrom takes precedence
if f.files != nil {
_, include := f.dirs[remote]
return include
// IncludeDirectory returns a function which checks whether this
// directory should be included in the sync or not.
func (f *Filter) IncludeDirectory(fs Fs) func(string) (bool, error) {
return func(remote string) (bool, error) {
remote = strings.Trim(remote, "/")
// first check if we need to remove directory based on
// the exclude file
excl, err := f.DirContainsExcludeFile(fs, remote)
if err != nil {
return false, err
}
if excl {
return false, nil
}
// filesFrom takes precedence
if f.files != nil {
_, include := f.dirs[remote]
return include, nil
}
remote += "/"
for _, rule := range f.dirRules.rules {
if rule.Match(remote) {
return rule.Include, nil
}
}
return true, nil
}
remote += "/"
for _, rule := range f.dirRules.rules {
if rule.Match(remote) {
return rule.Include
}
// DirContainsExcludeFile checks if exclude file is present in a
// directroy. If fs is nil, it works properly if ExcludeFile is an
// empty string (for testing).
func (f *Filter) DirContainsExcludeFile(fs Fs, remote string) (bool, error) {
if len(Config.Filter.ExcludeFile) > 0 {
exists, err := FileExists(fs, path.Join(remote, Config.Filter.ExcludeFile))
if err != nil {
return false, err
}
if exists {
return true, nil
}
}
return true
return false, nil
}
// Include returns whether this object should be included into the

View File

@ -164,7 +164,8 @@ type includeDirTest struct {
func testDirInclude(t *testing.T, f *Filter, tests []includeDirTest) {
for _, test := range tests {
got := f.IncludeDirectory(test.in)
got, err := f.IncludeDirectory(nil)(test.in)
require.NoError(t, err)
assert.Equal(t, test.want, got, test.in)
}
}

View File

@ -622,17 +622,20 @@ func ListDirSorted(fs Fs, includeAll bool, dir string) (entries DirEntries, err
if err != nil {
return nil, err
}
// This should happen only if exclude files lives in the
// starting directory, otherwise ListDirSorted should not be
// called.
if !includeAll && Config.Filter.ListContainsExcludeFile(entries) {
Debugf(dir, "Excluded from sync (and deletion) based on exclude file")
Debugf(dir, "Excluded from sync (and deletion)")
return nil, nil
}
return filterAndSortDir(entries, includeAll, dir, Config.Filter.IncludeObject, Config.Filter.IncludeDirectory)
return filterAndSortDir(entries, includeAll, dir, Config.Filter.IncludeObject, Config.Filter.IncludeDirectory(fs))
}
// filter (if required) and check the entries, then sort them
func filterAndSortDir(entries DirEntries, includeAll bool, dir string,
IncludeObject func(o Object) bool,
IncludeDirectory func(remote string) bool) (newEntries DirEntries, err error) {
IncludeDirectory func(remote string) (bool, error)) (newEntries DirEntries, err error) {
newEntries = entries[:0] // in place filter
prefix := ""
if dir != "" {
@ -649,9 +652,15 @@ func filterAndSortDir(entries DirEntries, includeAll bool, dir string,
Debugf(x, "Excluded from sync (and deletion)")
}
case Directory:
if !includeAll && !IncludeDirectory(x.Remote()) {
ok = false
Debugf(x, "Excluded from sync (and deletion)")
if !includeAll {
include, err := IncludeDirectory(x.Remote())
if err != nil {
return nil, err
}
if !include {
ok = false
Debugf(x, "Excluded from sync (and deletion)")
}
}
default:
return nil, errors.Errorf("unknown object type %T", entry)

View File

@ -22,8 +22,8 @@ func TestFilterAndSortIncludeAll(t *testing.T) {
includeObject := func(o Object) bool {
return o != oB
}
includeDirectory := func(remote string) bool {
return remote != "c"
includeDirectory := func(remote string) (bool, error) {
return remote != "c", nil
}
// no filter
newEntries, err := filterAndSortDir(entries, true, "", includeObject, includeDirectory)

View File

@ -349,6 +349,10 @@ func (dt DirTree) String() string {
func walkRDirTree(f Fs, startPath string, includeAll bool, maxLevel int, listR ListRFn) (DirTree, error) {
dirs := make(DirTree)
// Entries can come in arbitrary order. We use toPrune to keep
// all directories to exclude later.
toPrune := make(map[string]bool)
includeDirectory := Config.Filter.IncludeDirectory(f)
var mu sync.Mutex
err := listR(startPath, func(entries DirEntries) error {
mu.Lock()
@ -372,8 +376,21 @@ func walkRDirTree(f Fs, startPath string, includeAll bool, maxLevel int, listR L
} else {
Debugf(x, "Excluded from sync (and deletion)")
}
// Check if we need to prune a directory later.
if !includeAll && len(Config.Filter.ExcludeFile) > 0 {
basename := path.Base(x.Remote())
if basename == Config.Filter.ExcludeFile {
excludeDir := parentDir(x.Remote())
toPrune[excludeDir] = true
Debugf(basename, "Excluded from sync (and deletion) based on exclude file")
}
}
case Directory:
if includeAll || Config.Filter.IncludeDirectory(x.Remote()) {
inc, err := includeDirectory(x.Remote())
if err != nil {
return err
}
if includeAll || inc {
if maxLevel < 0 || slashes <= maxLevel-1 {
if slashes == maxLevel-1 {
// Just add the object if at maxLevel
@ -398,6 +415,10 @@ func walkRDirTree(f Fs, startPath string, includeAll bool, maxLevel int, listR L
if len(dirs) == 0 {
dirs[startPath] = nil
}
err = dirs.Prune(toPrune)
if err != nil {
return nil, err
}
dirs.Sort()
return dirs, nil
}