union: enable ListR when upstreams contain local

Enable fast list functions for union backend when:

- at least one of the upstreams supports fast list
- upstreams only consist of backends that support fast list and local backend.

Fixes #3000
This commit is contained in:
Max Sum 2020-05-13 20:10:35 +08:00 committed by GitHub
parent aba89e2737
commit 33d9310c49
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 23 additions and 7 deletions

View File

@ -19,6 +19,7 @@ import (
"github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fs/operations"
"github.com/rclone/rclone/fs/walk"
)
// Register with Fs
@ -597,17 +598,13 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
// Don't implement this unless you have a more efficient way
// of listing recursively that doing a directory traversal.
func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
for _, u := range f.upstreams {
if u.Features().ListR == nil {
return errors.Errorf("ListR Unsupported for branch: %s", u.Name())
}
}
var entriess [][]upstream.Entry
errs := Errors(make([]error, len(f.upstreams)))
var mutex sync.Mutex
multithread(len(f.upstreams), func(i int) {
u := f.upstreams[i]
err := u.Features().ListR(ctx, dir, func(entries fs.DirEntries) error {
var err error
callback := func(entries fs.DirEntries) error {
uEntries := make([]upstream.Entry, len(entries))
for j, e := range entries {
uEntries[j], _ = u.WrapEntry(e)
@ -616,7 +613,13 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
entriess = append(entriess, uEntries)
mutex.Unlock()
return nil
})
}
do := u.Features().ListR
if do != nil {
err = do(ctx, dir, callback)
} else {
err = walk.ListR(ctx, u, dir, true, -1, walk.ListAll, callback)
}
if err != nil {
errs[i] = errors.Wrap(err, u.Name())
return
@ -813,6 +816,19 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
features = features.Mask(f) // Mask all upstream fs
}
// Enable ListR when upstreams either support ListR or is local
// But not when all upstreams are local
if features.ListR == nil {
for _, u := range upstreams {
if u.Features().ListR != nil {
features.ListR = f.ListR
} else if !u.Features().IsLocal {
features.ListR = nil
break
}
}
}
f.features = features
// Get common intersection of hashes