community
/
rclone
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

filter: fix deadlock with errors on --files-from 

Before this change if doing a recursive directory listing with
`--files-from` if more than `--checkers` files errored (other than
file not found) then rclone would deadlock.

This fixes the problem by exiting on the first error.
This commit is contained in:
douchen 2023-06-10 22:53:08 +08:00 committed by GitHub
parent ae3ff50580
commit 7e2deffc62
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 17 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
fs/filter/filter.go
View File

@ -433,13 +433,13 @@ func (f *Filter) MakeListR(ctx context.Context, NewObject func(ctx context.Conte
var (
checkers = ci.Checkers
remotes = make(chan string, checkers)
g errgroup.Group
g, gCtx = errgroup.WithContext(ctx)
)
for i := 0; i < checkers; i++ {
g.Go(func() (err error) {
var entries = make(fs.DirEntries, 1)
for remote := range remotes {
entries[0], err = NewObject(ctx, remote)
entries[0], err = NewObject(gCtx, remote)
if err == fs.ErrorObjectNotFound {
// Skip files that are not found
} else if err != nil {
@ -454,8 +454,13 @@ func (f *Filter) MakeListR(ctx context.Context, NewObject func(ctx context.Conte
return nil
})
}
outer:
for remote := range f.files {
remotes <- remote
select {
case remotes <- remote:
case <-gCtx.Done():
break outer
}
}
close(remotes)
return g.Wait()

9
fs/filter/filter_test.go
View File

@ -360,6 +360,15 @@ func TestNewFilterMakeListR(t *testing.T) {
require.NoError(t, f.AddFile("error"))
err = listR(context.Background(), "", listRcallback)
require.EqualError(t, err, assert.AnError.Error())
// The checker will exit by the error above
ci := fs.GetConfig(context.Background())
ci.Checkers = 1
// Now check an error is returned from NewObject
require.NoError(t, f.AddFile("error"))
err = listR(context.Background(), "", listRcallback)
require.EqualError(t, err, assert.AnError.Error())
}
func TestNewFilterMinSize(t *testing.T) {