community
/
rclone
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

bisync: dry runs no longer commit filter changes 

Fixed an issue causing dry runs to inadvertently commit filter changes
https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=1.%20Dry%20runs%20are%20not%20completely%20dry
This commit is contained in:
nielash 2023-07-11 04:35:01 -04:00 committed by Nick Craig-Wood
parent 4f1eafb044
commit f4dd86238d
9 changed files with 58 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cmd/bisync/bisync_test.go
View File

@ -1163,6 +1163,10 @@ func (b *bisyncTest) newReplacer(mangle bool) *strings.Replacer {
b.workDir + slash, "{workdir/}",
b.path1, "{path1/}",
b.path2, "{path2/}",
"//?/" + strings.TrimSuffix(strings.Replace(b.path1, slash, "/", -1), "/"), "{path1}", // fix windows-specific issue
"//?/" + strings.TrimSuffix(strings.Replace(b.path2, slash, "/", -1), "/"), "{path2}",
strings.TrimSuffix(b.path1, slash), "{path1}", // ensure it's still recognized without trailing slash
strings.TrimSuffix(b.path2, slash), "{path2}",
b.sessionName, "{session}",
}
if fixSlash {

4
cmd/bisync/cmd.go
View File

@ -210,9 +210,13 @@ func (opt *Options) applyFilters(ctx context.Context) (context.Context, error) {
}
if opt.Resync {
if opt.DryRun {
fs.Infof(nil, "Skipped storing filters file hash to %s as --dry-run is set", hashFile)
} else {
fs.Infof(nil, "Storing filters file hash to %s", hashFile)
if err := os.WriteFile(hashFile, []byte(gotHash), bilib.PermSecure); err != nil {
return ctx, err
}
}
}

5
cmd/bisync/testdata/test_filtersfile_checks/golden/_testdir_path1.._testdir_path2.path1.lst-dry vendored Normal file
View File

@ -0,0 +1,5 @@
# bisync listing v1 from test
- 109 md5:294d25b294ff26a5243dba914ac3fbf7 - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file2.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "subdir/file20.txt"

5
cmd/bisync/testdata/test_filtersfile_checks/golden/_testdir_path1.._testdir_path2.path1.lst-dry-new vendored Normal file
View File

@ -0,0 +1,5 @@
# bisync listing v1 from test
- 109 md5:294d25b294ff26a5243dba914ac3fbf7 - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file2.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "subdir/file20.txt"

5
cmd/bisync/testdata/test_filtersfile_checks/golden/_testdir_path1.._testdir_path2.path2.lst-dry vendored Normal file
View File

@ -0,0 +1,5 @@
# bisync listing v1 from test
- 109 md5:294d25b294ff26a5243dba914ac3fbf7 - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file2.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "subdir/file20.txt"

5
cmd/bisync/testdata/test_filtersfile_checks/golden/_testdir_path1.._testdir_path2.path2.lst-dry-new vendored Normal file
View File

@ -0,0 +1,5 @@
# bisync listing v1 from test
- 109 md5:294d25b294ff26a5243dba914ac3fbf7 - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file2.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "subdir/file20.txt"

18
cmd/bisync/testdata/test_filtersfile_checks/golden/test.log vendored
View File

@ -58,3 +58,21 @@ INFO : Using filters file {workdir/}filtersfile.txt
ERROR : Bisync critical error: filters file has changed (must run --resync): {workdir/}filtersfile.txt
ERROR : Bisync aborted. Must run --resync to recover.
Bisync error: bisync aborted
(18) : test 8. run with filters-file and resync and dry-run. should do the dry-run but still cause next non-resync run to abort.
(19) : bisync filters-file={workdir/}filtersfile.txt resync dry-run
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Using filters file {workdir/}filtersfile.txt
INFO : Skipped storing filters file hash to {workdir/}filtersfile.txt.md5 as --dry-run is set
INFO : Copying unique Path2 files to Path1
INFO : Resynching Path1 to Path2
INFO : Resync updating listings
INFO : Bisync successful
(20) : test 9. run with filters-file alone. should abort.
(21) : bisync filters-file={workdir/}filtersfile.txt
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Using filters file {workdir/}filtersfile.txt
ERROR : Bisync critical error: filters file has changed (must run --resync): {workdir/}filtersfile.txt
ERROR : Bisync aborted. Must run --resync to recover.
Bisync error: bisync aborted

6
cmd/bisync/testdata/test_filtersfile_checks/scenario.txt vendored
View File

@ -34,3 +34,9 @@ copy-as {datadir/}filtersfile2.txt {workdir/} filtersfile.txt
test 7. run with filters-file alone. should abort.
bisync filters-file={workdir/}filtersfile.txt
test 8. run with filters-file and resync and dry-run. should do the dry-run but still cause next non-resync run to abort.
bisync filters-file={workdir/}filtersfile.txt resync dry-run
test 9. run with filters-file alone. should abort.
bisync filters-file={workdir/}filtersfile.txt

6
docs/content/bisync.md
View File

@ -1102,3 +1102,9 @@ See the _Dual Shadow Method_ section in the
Also note a number of academic publications by
[Benjamin Pierce](http://www.cis.upenn.edu/%7Ebcpierce/papers/index.shtml#File%20Synchronization)
about _Unison_ and synchronization in general.
## Changelog
### `v1.64`
* Fixed an [issue](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=1.%20Dry%20runs%20are%20not%20completely%20dry)
causing dry runs to inadvertently commit filter changes