diff --git a/cmd/bisync/deltas.go b/cmd/bisync/deltas.go index ccd43edbe..6e0542dfb 100644 --- a/cmd/bisync/deltas.go +++ b/cmd/bisync/deltas.go @@ -327,6 +327,7 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change if !in2 { b.indent("Path2", p2, "Queue delete") delete2.Add(file) + copy1to2.Add(file) } else if d2.is(deltaOther) { b.indent("Path2", p1, "Queue copy to Path1") copy2to1.Add(file) @@ -351,6 +352,7 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change // Deleted b.indent("Path1", p1, "Queue delete") delete1.Add(file) + copy2to1.Add(file) } } @@ -380,25 +382,17 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change } if delete1.NotEmpty() { - changes1 = true - b.indent("", "Path1", "Do queued deletes on") - err = b.fastDelete(ctx, b.fs1, delete1, "delete1") - if err != nil { + if err = b.saveQueue(delete1, "delete1"); err != nil { return } - //propagate deletions of empty dirs from path2 to path1 (if --create-empty-src-dirs) b.syncEmptyDirs(ctx, b.fs1, delete1, dirs1, "remove") } if delete2.NotEmpty() { - changes2 = true - b.indent("", "Path2", "Do queued deletes on") - err = b.fastDelete(ctx, b.fs2, delete2, "delete2") - if err != nil { + if err = b.saveQueue(delete2, "delete2"); err != nil { return } - //propagate deletions of empty dirs from path1 to path2 (if --create-empty-src-dirs) b.syncEmptyDirs(ctx, b.fs2, delete2, dirs2, "remove") } diff --git a/cmd/bisync/queue.go b/cmd/bisync/queue.go index 701c8a922..d213ff74a 100644 --- a/cmd/bisync/queue.go +++ b/cmd/bisync/queue.go @@ -24,39 +24,11 @@ func (b *bisyncRun) fastCopy(ctx context.Context, fsrc, fdst fs.Fs, files bilib. } } - return sync.CopyDir(ctxCopy, fdst, fsrc, b.opt.CreateEmptySrcDirs) -} - -func (b *bisyncRun) fastDelete(ctx context.Context, f fs.Fs, files bilib.Names, queueName string) error { - if err := b.saveQueue(files, queueName); err != nil { - return err - } - - transfers := fs.GetConfig(ctx).Transfers - - ctxRun, filterDelete := filter.AddConfig(b.opt.setDryRun(ctx)) - - for _, file := range files.ToList() { - if err := filterDelete.AddFile(file); err != nil { - return err - } - } - - objChan := make(fs.ObjectsChan, transfers) - errChan := make(chan error, 1) - go func() { - errChan <- operations.DeleteFiles(ctxRun, objChan) - }() - err := operations.ListFn(ctxRun, f, func(obj fs.Object) { - remote := obj.Remote() - if files.Has(remote) { - objChan <- obj - } - }) - close(objChan) - opErr := <-errChan - if err == nil { - err = opErr + var err error + if b.opt.Resync { + err = sync.CopyDir(ctxCopy, fdst, fsrc, b.opt.CreateEmptySrcDirs) + } else { + err = sync.Sync(ctxCopy, fdst, fsrc, b.opt.CreateEmptySrcDirs) } return err } @@ -75,8 +47,9 @@ func (b *bisyncRun) syncEmptyDirs(ctx context.Context, dst fs.Fs, candidates bil var direrr error if dirsList.has(s) { //make sure it's a dir, not a file if operation == "remove" { - //note: we need to use Rmdirs instead of Rmdir because directories will fail to delete if they have other empty dirs inside of them. - direrr = operations.Rmdirs(ctx, dst, s, false) + // directories made empty by the sync will have already been deleted during the sync + // this just catches the already-empty ones (excluded from sync by --files-from filter) + direrr = operations.TryRmdir(ctx, dst, s) } else if operation == "make" { direrr = operations.Mkdir(ctx, dst, s) } else { diff --git a/cmd/bisync/testdata/test_changes/golden/_testdir_path1.._testdir_path2.copy1to2.que b/cmd/bisync/testdata/test_changes/golden/_testdir_path1.._testdir_path2.copy1to2.que index 38a6601db..5725b8184 100644 --- a/cmd/bisync/testdata/test_changes/golden/_testdir_path1.._testdir_path2.copy1to2.que +++ b/cmd/bisync/testdata/test_changes/golden/_testdir_path1.._testdir_path2.copy1to2.que @@ -1,4 +1,5 @@ "file11.txt" "file2.txt" +"file4.txt" "file5.txt..path1" "file7.txt" diff --git a/cmd/bisync/testdata/test_changes/golden/_testdir_path1.._testdir_path2.copy2to1.que b/cmd/bisync/testdata/test_changes/golden/_testdir_path1.._testdir_path2.copy2to1.que index 4146ce3e0..3b79e78bd 100644 --- a/cmd/bisync/testdata/test_changes/golden/_testdir_path1.._testdir_path2.copy2to1.que +++ b/cmd/bisync/testdata/test_changes/golden/_testdir_path1.._testdir_path2.copy2to1.que @@ -1,4 +1,5 @@ "file1.txt" "file10.txt" +"file3.txt" "file5.txt..path2" "file6.txt" diff --git a/cmd/bisync/testdata/test_changes/golden/test.log b/cmd/bisync/testdata/test_changes/golden/test.log index 222b29758..f77d40340 100644 --- a/cmd/bisync/testdata/test_changes/golden/test.log +++ b/cmd/bisync/testdata/test_changes/golden/test.log @@ -88,8 +88,6 @@ INFO : - Path2 Queue copy to Path1 - {path1/}file10.txt INFO : - Path1 Queue delete - {path1/}file3.txt INFO : - Path2 Do queued copies to - Path1 INFO : - Path1 Do queued copies to - Path2 -INFO : - Do queued deletes on - Path1 -INFO : - Do queued deletes on - Path2 INFO : Updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful diff --git a/cmd/bisync/testdata/test_createemptysrcdirs/golden/test.log b/cmd/bisync/testdata/test_createemptysrcdirs/golden/test.log index b8799524a..c63c8850c 100644 --- a/cmd/bisync/testdata/test_createemptysrcdirs/golden/test.log +++ b/cmd/bisync/testdata/test_createemptysrcdirs/golden/test.log @@ -77,7 +77,7 @@ INFO : - Path2 File was deleted - subdir INFO : Path2: 1 changes: 0 new, 0 newer, 0 older, 1 deleted INFO : Applying changes INFO : - Path2 Queue delete - {path2/}RCLONE_TEST -INFO : - Do queued deletes on - Path2 +INFO : - Path1 Do queued copies to - Path2 INFO : Updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful @@ -121,7 +121,7 @@ INFO : Path1: 1 changes: 0 new, 0 newer, 0 older, 1 deleted INFO : Path2 checking for diffs INFO : Applying changes INFO : - Path2 Queue delete - {path2/}subdir -INFO : - Do queued deletes on - Path2 +INFO : - Path1 Do queued copies to - Path2 INFO : subdir: Removing directory INFO : Updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" diff --git a/cmd/bisync/testdata/test_dry_run/golden/_testdir_path1.._testdir_path2.copy1to2.que b/cmd/bisync/testdata/test_dry_run/golden/_testdir_path1.._testdir_path2.copy1to2.que index 38a6601db..5725b8184 100644 --- a/cmd/bisync/testdata/test_dry_run/golden/_testdir_path1.._testdir_path2.copy1to2.que +++ b/cmd/bisync/testdata/test_dry_run/golden/_testdir_path1.._testdir_path2.copy1to2.que @@ -1,4 +1,5 @@ "file11.txt" "file2.txt" +"file4.txt" "file5.txt..path1" "file7.txt" diff --git a/cmd/bisync/testdata/test_dry_run/golden/_testdir_path1.._testdir_path2.copy2to1.que b/cmd/bisync/testdata/test_dry_run/golden/_testdir_path1.._testdir_path2.copy2to1.que index 4146ce3e0..3b79e78bd 100644 --- a/cmd/bisync/testdata/test_dry_run/golden/_testdir_path1.._testdir_path2.copy2to1.que +++ b/cmd/bisync/testdata/test_dry_run/golden/_testdir_path1.._testdir_path2.copy2to1.que @@ -1,4 +1,5 @@ "file1.txt" "file10.txt" +"file3.txt" "file5.txt..path2" "file6.txt" diff --git a/cmd/bisync/testdata/test_dry_run/golden/dryrun._testdir_path1.._testdir_path2.copy1to2.que b/cmd/bisync/testdata/test_dry_run/golden/dryrun._testdir_path1.._testdir_path2.copy1to2.que index 38a6601db..5725b8184 100644 --- a/cmd/bisync/testdata/test_dry_run/golden/dryrun._testdir_path1.._testdir_path2.copy1to2.que +++ b/cmd/bisync/testdata/test_dry_run/golden/dryrun._testdir_path1.._testdir_path2.copy1to2.que @@ -1,4 +1,5 @@ "file11.txt" "file2.txt" +"file4.txt" "file5.txt..path1" "file7.txt" diff --git a/cmd/bisync/testdata/test_dry_run/golden/dryrun._testdir_path1.._testdir_path2.copy2to1.que b/cmd/bisync/testdata/test_dry_run/golden/dryrun._testdir_path1.._testdir_path2.copy2to1.que index 4146ce3e0..3b79e78bd 100644 --- a/cmd/bisync/testdata/test_dry_run/golden/dryrun._testdir_path1.._testdir_path2.copy2to1.que +++ b/cmd/bisync/testdata/test_dry_run/golden/dryrun._testdir_path1.._testdir_path2.copy2to1.que @@ -1,4 +1,5 @@ "file1.txt" "file10.txt" +"file3.txt" "file5.txt..path2" "file6.txt" diff --git a/cmd/bisync/testdata/test_dry_run/golden/test.log b/cmd/bisync/testdata/test_dry_run/golden/test.log index 13bd9654a..3d749965b 100644 --- a/cmd/bisync/testdata/test_dry_run/golden/test.log +++ b/cmd/bisync/testdata/test_dry_run/golden/test.log @@ -106,15 +106,13 @@ INFO : - Path1 Queue delete - {path1/}file3.txt INFO : - Path2 Do queued copies to - Path1 NOTICE: file1.txt: Skipped copy as --dry-run is set (size 19) NOTICE: file10.txt: Skipped copy as --dry-run is set (size 19) +NOTICE: file3.txt: Skipped delete as --dry-run is set (size 0) NOTICE: file6.txt: Skipped copy as --dry-run is set (size 19) INFO : - Path1 Do queued copies to - Path2 NOTICE: file11.txt: Skipped copy as --dry-run is set (size 19) NOTICE: file2.txt: Skipped copy as --dry-run is set (size 13) -NOTICE: file7.txt: Skipped copy as --dry-run is set (size 19) -INFO : - Do queued deletes on - Path1 -NOTICE: file3.txt: Skipped delete as --dry-run is set (size 0) -INFO : - Do queued deletes on - Path2 NOTICE: file4.txt: Skipped delete as --dry-run is set (size 0) +NOTICE: file7.txt: Skipped copy as --dry-run is set (size 19) INFO : Updating listings INFO : Bisync successful (32) : copy-listings dryrun @@ -159,8 +157,6 @@ INFO : - Path2 Queue copy to Path1 - {path1/}file10.txt INFO : - Path1 Queue delete - {path1/}file3.txt INFO : - Path2 Do queued copies to - Path1 INFO : - Path1 Do queued copies to - Path2 -INFO : - Do queued deletes on - Path1 -INFO : - Do queued deletes on - Path2 INFO : Updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful diff --git a/cmd/bisync/testdata/test_extended_filenames/golden/_testdir_path1.._testdir_path2.copy2to1.que b/cmd/bisync/testdata/test_extended_filenames/golden/_testdir_path1.._testdir_path2.copy2to1.que index eece110c1..5c39d3955 100644 --- a/cmd/bisync/testdata/test_extended_filenames/golden/_testdir_path1.._testdir_path2.copy2to1.que +++ b/cmd/bisync/testdata/test_extended_filenames/golden/_testdir_path1.._testdir_path2.copy2to1.que @@ -1,7 +1,9 @@ "New_top_level_mañana_funcionará.txt" "file_enconde_mañana_funcionará.txt" +"filename_contains_ࢺ_.txt" "subdir with␊white space.txt/file2 with␊white space.txt" "subdir_rawchars_␙_\x81_\xfe/file3_␙_\x81_\xfe" "subdir_with_ࢺ_/filechangedbothpaths_ࢺ_.txt..path2" +"subdir_with_ࢺ_/filename_contains_ě_.txt" "subdir_with_ࢺ_/filename_contains_ࢺ_p2s.txt" "Русский.txt" diff --git a/cmd/bisync/testdata/test_extended_filenames/golden/test.log b/cmd/bisync/testdata/test_extended_filenames/golden/test.log index 5a7f6412a..d1db495d9 100644 --- a/cmd/bisync/testdata/test_extended_filenames/golden/test.log +++ b/cmd/bisync/testdata/test_extended_filenames/golden/test.log @@ -84,7 +84,6 @@ INFO : - Path1 Queue delete - {path1/}subdir_with_ࢺ INFO : - Path2 Queue copy to Path1 - {path1/}subdir_with_ࢺ_/filename_contains_ࢺ_p2s.txt INFO : - Path2 Do queued copies to - Path1 INFO : - Path1 Do queued copies to - Path2 -INFO : - Do queued deletes on - Path1 INFO : Updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful diff --git a/cmd/bisync/testdata/test_max_delete_path1/golden/_testdir_path1.._testdir_path2.copy1to2.que b/cmd/bisync/testdata/test_max_delete_path1/golden/_testdir_path1.._testdir_path2.copy1to2.que new file mode 100644 index 000000000..9c041c2ba --- /dev/null +++ b/cmd/bisync/testdata/test_max_delete_path1/golden/_testdir_path1.._testdir_path2.copy1to2.que @@ -0,0 +1,5 @@ +"file1.txt" +"file2.txt" +"file3.txt" +"file4.txt" +"file5.txt" diff --git a/cmd/bisync/testdata/test_max_delete_path1/golden/test.log b/cmd/bisync/testdata/test_max_delete_path1/golden/test.log index c7c208211..7d83e9150 100644 --- a/cmd/bisync/testdata/test_max_delete_path1/golden/test.log +++ b/cmd/bisync/testdata/test_max_delete_path1/golden/test.log @@ -49,7 +49,7 @@ INFO : - Path2 Queue delete - {path2/}file2.txt INFO : - Path2 Queue delete - {path2/}file3.txt INFO : - Path2 Queue delete - {path2/}file4.txt INFO : - Path2 Queue delete - {path2/}file5.txt -INFO : - Do queued deletes on - Path2 +INFO : - Path1 Do queued copies to - Path2 INFO : Updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful diff --git a/cmd/bisync/testdata/test_max_delete_path2_force/golden/_testdir_path1.._testdir_path2.copy2to1.que b/cmd/bisync/testdata/test_max_delete_path2_force/golden/_testdir_path1.._testdir_path2.copy2to1.que new file mode 100644 index 000000000..9c041c2ba --- /dev/null +++ b/cmd/bisync/testdata/test_max_delete_path2_force/golden/_testdir_path1.._testdir_path2.copy2to1.que @@ -0,0 +1,5 @@ +"file1.txt" +"file2.txt" +"file3.txt" +"file4.txt" +"file5.txt" diff --git a/cmd/bisync/testdata/test_max_delete_path2_force/golden/test.log b/cmd/bisync/testdata/test_max_delete_path2_force/golden/test.log index 58da0c3a3..23cc350c4 100644 --- a/cmd/bisync/testdata/test_max_delete_path2_force/golden/test.log +++ b/cmd/bisync/testdata/test_max_delete_path2_force/golden/test.log @@ -49,7 +49,7 @@ INFO : - Path1 Queue delete - {path1/}file2.txt INFO : - Path1 Queue delete - {path1/}file3.txt INFO : - Path1 Queue delete - {path1/}file4.txt INFO : - Path1 Queue delete - {path1/}file5.txt -INFO : - Do queued deletes on - Path1 +INFO : - Path2 Do queued copies to - Path1 INFO : Updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful diff --git a/cmd/bisync/testdata/test_rmdirs/golden/_testdir_path1.._testdir_path2.copy1to2.que b/cmd/bisync/testdata/test_rmdirs/golden/_testdir_path1.._testdir_path2.copy1to2.que new file mode 100644 index 000000000..a49deb16c --- /dev/null +++ b/cmd/bisync/testdata/test_rmdirs/golden/_testdir_path1.._testdir_path2.copy1to2.que @@ -0,0 +1 @@ +"subdir/file20.txt" diff --git a/cmd/bisync/testdata/test_rmdirs/golden/test.log b/cmd/bisync/testdata/test_rmdirs/golden/test.log index 936af897b..b770c2482 100644 --- a/cmd/bisync/testdata/test_rmdirs/golden/test.log +++ b/cmd/bisync/testdata/test_rmdirs/golden/test.log @@ -21,7 +21,7 @@ INFO : Path1: 1 changes: 0 new, 0 newer, 0 older, 1 deleted INFO : Path2 checking for diffs INFO : Applying changes INFO : - Path2 Queue delete - {path2/}subdir/file20.txt -INFO : - Do queued deletes on - Path2 +INFO : - Path1 Do queued copies to - Path2 INFO : Updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful diff --git a/docs/content/bisync.md b/docs/content/bisync.md index be03f8cb8..94dc815e3 100644 --- a/docs/content/bisync.md +++ b/docs/content/bisync.md @@ -552,11 +552,17 @@ and use `--resync` when you need to switch. ### Renamed directories -Renaming a folder on the Path1 side results in deleting all files on +By default, renaming a folder on the Path1 side results in deleting all files on the Path2 side and then copying all files again from Path1 to Path2. Bisync sees this as all files in the old directory name as deleted and all files in the new directory name as new. -Currently, the most effective and efficient method of renaming a directory + +A recommended solution is to use [`--track-renames`](/docs/#track-renames), +which is now supported in bisync as of `rclone v1.65`. +Note that `--track-renames` is not available during `--resync`, +as `--resync` does not delete anything (`--track-renames` only supports `sync`, not `copy`.) + +Otherwise, the most effective and efficient method of renaming a directory is to rename it to the same name on both sides. (As of `rclone v1.64`, a `--resync` is no longer required after doing so, as bisync will automatically detect that Path1 and Path2 are in agreement.) @@ -1263,6 +1269,10 @@ about _Unison_ and synchronization in general. ## Changelog +### `v1.65` +* Copies and deletes are now handled in one operation instead of two +* `--track-renames` and `--backup-dir` are now supported + ### `v1.64` * Fixed an [issue](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=1.%20Dry%20runs%20are%20not%20completely%20dry) causing dry runs to inadvertently commit filter changes