rmdirs: remove directories concurrently controlled by --checkers

See: https://forum.rclone.org/t/how-to-list-empty-directories-recursively/40995
This commit is contained in:
Nick Craig-Wood 2023-08-17 11:05:12 +01:00
parent dc803b572c
commit 03aab1a123
3 changed files with 73 additions and 13 deletions

View File

@ -35,7 +35,10 @@ empty directories in. For example the [delete](/commands/rclone_delete/)
command will delete files but leave the directory structure (unless
used with option ` + "`--rmdirs`" + `).
To delete a path and any objects in it, use [purge](/commands/rclone_purge/)
This will delete ` + "`--checkers`" + ` directories concurrently so
if you have thousands of empty directories consider increasing this number.
To delete a path and any objects in it, use the [purge](/commands/rclone_purge/)
command.
`,
Annotations: map[string]string{

View File

@ -1551,28 +1551,69 @@ func Rmdirs(ctx context.Context, f fs.Fs, dir string, leaveRoot bool) error {
if err != nil {
return fmt.Errorf("failed to rmdirs: %w", err)
}
// Now delete the empty directories, starting from the longest path
var toDelete []string
// Group directories to delete by level
var toDelete [][]string
for dir, empty := range dirEmpty {
if empty {
toDelete = append(toDelete, dir)
}
}
sort.Strings(toDelete)
for i := len(toDelete) - 1; i >= 0; i-- {
dir := toDelete[i]
// If a filter matches the directory then that
// directory is a candidate for deletion
if !fi.IncludeRemote(dir + "/") {
if fi.IncludeRemote(dir + "/") {
level := strings.Count(dir, "/") + 1
// The root directory "" is at the top level
if dir == "" {
level = 0
}
if len(toDelete) < level+1 {
toDelete = append(toDelete, make([][]string, level+1-len(toDelete))...)
}
toDelete[level] = append(toDelete[level], dir)
}
}
}
var (
errMu sync.Mutex
errCount int
lastError error
)
// Delete all directories at the same level in parallel
for level := len(toDelete) - 1; level >= 0; level-- {
dirs := toDelete[level]
if len(dirs) == 0 {
continue
}
err = TryRmdir(ctx, f, dir)
fs.Debugf(nil, "removing %d level %d directories", len(dirs), level)
sort.Strings(dirs)
g, gCtx := errgroup.WithContext(ctx)
g.SetLimit(ci.Checkers)
for _, dir := range dirs {
// End early if error
if gCtx.Err() != nil {
break
}
dir := dir
g.Go(func() error {
err := TryRmdir(gCtx, f, dir)
if err != nil {
err = fs.CountError(err)
fs.Errorf(dir, "Failed to rmdir: %v", err)
errMu.Lock()
lastError = err
errCount += 1
errMu.Unlock()
}
return nil // don't return errors, just count them
})
}
err := g.Wait()
if err != nil {
return err
}
}
if lastError != nil {
return fmt.Errorf("failed to remove %d directories: last error: %w", errCount, lastError)
}
return nil
}

View File

@ -702,6 +702,22 @@ func TestRmdirsNoLeaveRoot(t *testing.T) {
fs.GetModifyWindow(ctx, r.Fremote),
)
// Delete the files so we can remove everything including the root
for _, file := range []fstest.Item{file1, file2} {
o, err := r.Fremote.NewObject(ctx, file.Path)
require.NoError(t, err)
require.NoError(t, o.Remove(ctx))
}
require.NoError(t, operations.Rmdirs(ctx, r.Fremote, "", false))
fstest.CheckListingWithPrecision(
t,
r.Fremote,
[]fstest.Item{},
[]string{},
fs.GetModifyWindow(ctx, r.Fremote),
)
}
func TestRmdirsLeaveRoot(t *testing.T) {