From f7f465182832fe4448b1b638088bb8e96a8af029 Mon Sep 17 00:00:00 2001 From: nielash Date: Sun, 8 Oct 2023 23:16:23 -0400 Subject: [PATCH] bisync: handle unicode and case normalization consistently - mostly-fixes #7270 Before this change, Bisync sometimes normalized NFD to NFC and sometimes did not, causing errors in some scenarios (particularly for users of macOS). It was similarly inconsistent in its handling of case-insensitivity. There were three main places where Bisync should have normalized, but didn't: 1. When building the list of files that need to be transferred during --resync 2. When building the list of deltas during a non-resync 3. When comparing Path1 to Path2 during --check-sync After this change, 1 and 3 are resolved, and bisync supports --no-unicode-normalization and --ignore-case-sync in the same way as sync. 2 will be addressed in a future update. --- cmd/bisync/bisync_test.go | 28 +++- cmd/bisync/deltas.go | 10 +- cmd/bisync/listing.go | 68 +++++++-- cmd/bisync/march.go | 4 - cmd/bisync/operations.go | 21 ++- cmd/bisync/queue.go | 31 +++- .../_testdir_path1.._testdir_path2.path1.lst | 17 +-- ...estdir_path1.._testdir_path2.path1.lst-new | 1 - ...estdir_path1.._testdir_path2.path1.lst-old | 17 +-- .../_testdir_path1.._testdir_path2.path2.lst | 17 +-- ...estdir_path1.._testdir_path2.path2.lst-new | 1 - ...estdir_path1.._testdir_path2.path2.lst-old | 17 +-- .../test_check_access_filters/golden/test.log | 2 +- ...testdir_path1.._testdir_path2.copy1to2.que | 3 + ...testdir_path1.._testdir_path2.copy2to1.que | 3 + .../_testdir_path1.._testdir_path2.path1.lst | 5 + ...estdir_path1.._testdir_path2.path1.lst-new | 5 + ...estdir_path1.._testdir_path2.path1.lst-old | 5 + .../_testdir_path1.._testdir_path2.path2.lst | 5 + ...estdir_path1.._testdir_path2.path2.lst-new | 5 + ...estdir_path1.._testdir_path2.path2.lst-old | 5 + ..._path1.._testdir_path2.resync-copy2to1.que | 2 + .../test_normalization/golden/test.log | 144 ++++++++++++++++++ .../test_normalization/initial/RCLONE_TEST | 1 + .../test_normalization/initial/file1.txt | 0 .../filename_contains_ě_.txt | 7 + .../filename_contains_ࢺ_.txt | 7 + .../測試_Русский_ _ _ě_áñ/測試_check file | 0 .../test_normalization/modfiles/file1.txt | 1 + .../modfiles/測試_filtersfile.txt | 5 + .../testdata/test_normalization/scenario.txt | 53 +++++++ docs/content/bisync.md | 20 ++- 32 files changed, 441 insertions(+), 69 deletions(-) create mode 100644 cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.copy1to2.que create mode 100644 cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.copy2to1.que create mode 100644 cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path1.lst create mode 100644 cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path1.lst-new create mode 100644 cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path1.lst-old create mode 100644 cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst create mode 100644 cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst-new create mode 100644 cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst-old create mode 100644 cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.resync-copy2to1.que create mode 100644 cmd/bisync/testdata/test_normalization/golden/test.log create mode 100644 cmd/bisync/testdata/test_normalization/initial/RCLONE_TEST create mode 100644 cmd/bisync/testdata/test_normalization/initial/file1.txt create mode 100644 cmd/bisync/testdata/test_normalization/initial/測試_Русский_ _ _ě_áñ/filename_contains_ě_.txt create mode 100644 cmd/bisync/testdata/test_normalization/initial/測試_Русский_ _ _ě_áñ/filename_contains_ࢺ_.txt create mode 100644 cmd/bisync/testdata/test_normalization/initial/測試_Русский_ _ _ě_áñ/測試_check file create mode 100644 cmd/bisync/testdata/test_normalization/modfiles/file1.txt create mode 100644 cmd/bisync/testdata/test_normalization/modfiles/測試_filtersfile.txt create mode 100644 cmd/bisync/testdata/test_normalization/scenario.txt diff --git a/cmd/bisync/bisync_test.go b/cmd/bisync/bisync_test.go index 11a0862c3..185981a2c 100644 --- a/cmd/bisync/bisync_test.go +++ b/cmd/bisync/bisync_test.go @@ -98,10 +98,13 @@ var logHoppers = []string{ // subdirectories. The order inconsistency initially showed up in the // listings and triggered reordering of log messages, but the actual // files will in fact match. - `ERROR : - +Access test failed: Path[12] file not found in Path[12] - .*`, + `.* +.....Access test failed: Path[12] file not found in Path[12].*`, // Test case `resync` suffered from the order of queued copies. `(?:INFO |NOTICE): - Path2 Resync will copy to Path1 +- .*`, + + // Test case `normalization` can have random order of fix-case files. + `(?:INFO |NOTICE): .*: Fixed case by renaming to: .*`, } // Some log lines can contain Windows path separator that must be @@ -546,6 +549,16 @@ func (b *bisyncTest) runTestStep(ctx context.Context, line string) (err error) { case "copy-as": b.checkArgs(args, 3, 3) return b.copyFile(ctx, args[1], args[2], args[3]) + case "copy-as-NFC": + b.checkArgs(args, 3, 3) + ci.NoUnicodeNormalization = true + ci.FixCase = true + return b.copyFile(ctx, args[1], norm.NFC.String(args[2]), norm.NFC.String(args[3])) + case "copy-as-NFD": + b.checkArgs(args, 3, 3) + ci.NoUnicodeNormalization = true + ci.FixCase = true + return b.copyFile(ctx, args[1], norm.NFD.String(args[2]), norm.NFD.String(args[3])) case "copy-dir", "sync-dir": b.checkArgs(args, 2, 2) if fsrc, err = cache.Get(ctx, args[1]); err != nil { @@ -565,6 +578,9 @@ func (b *bisyncTest) runTestStep(ctx context.Context, line string) (err error) { b.checkArgs(args, 1, 1) return b.listSubdirs(ctx, args[1]) case "bisync": + ci.NoUnicodeNormalization = false + ci.IgnoreCaseSync = false + // ci.FixCase = true return b.runBisync(ctx, args[1:]) case "test-func": b.TestFn = testFunc @@ -668,6 +684,16 @@ func (b *bisyncTest) runBisync(ctx context.Context, args []string) (err error) { fs2 = addSubdir(b.path2, val) case "ignore-listing-checksum": opt.IgnoreListingChecksum = true + case "no-norm": + ci.NoUnicodeNormalization = true + ci.IgnoreCaseSync = false + case "norm": + ci.NoUnicodeNormalization = false + ci.IgnoreCaseSync = true + case "fix-case": + ci.NoUnicodeNormalization = false + ci.IgnoreCaseSync = true + ci.FixCase = true default: return fmt.Errorf("invalid bisync option %q", arg) } diff --git a/cmd/bisync/deltas.go b/cmd/bisync/deltas.go index 3167dfb00..d0c0cd8aa 100644 --- a/cmd/bisync/deltas.go +++ b/cmd/bisync/deltas.go @@ -381,11 +381,17 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change } } + // find alternate names according to normalization settings + altNames1to2 := bilib.Names{} + altNames2to1 := bilib.Names{} + b.findAltNames(ctx, b.fs1, copy2to1, b.newListing1, altNames2to1) + b.findAltNames(ctx, b.fs2, copy1to2, b.newListing2, altNames1to2) + // Do the batch operation if copy2to1.NotEmpty() { changes1 = true b.indent("Path2", "Path1", "Do queued copies to") - results2to1, err = b.fastCopy(ctx, b.fs2, b.fs1, copy2to1, "copy2to1") + results2to1, err = b.fastCopy(ctx, b.fs2, b.fs1, copy2to1, "copy2to1", altNames2to1) if err != nil { return } @@ -397,7 +403,7 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change if copy1to2.NotEmpty() { changes2 = true b.indent("Path1", "Path2", "Do queued copies to") - results1to2, err = b.fastCopy(ctx, b.fs1, b.fs2, copy1to2, "copy1to2") + results1to2, err = b.fastCopy(ctx, b.fs1, b.fs2, copy1to2, "copy1to2", altNames1to2) if err != nil { return } diff --git a/cmd/bisync/listing.go b/cmd/bisync/listing.go index 7875036fa..f94883d97 100644 --- a/cmd/bisync/listing.go +++ b/cmd/bisync/listing.go @@ -20,6 +20,7 @@ import ( "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/fs/operations" "golang.org/x/exp/slices" + "golang.org/x/text/unicode/norm" ) // ListingHeader defines first line of a listing @@ -400,6 +401,18 @@ func ConvertPrecision(Modtime time.Time, dst fs.Fs) time.Time { return Modtime } +// ApplyTransforms handles unicode and case normalization +func ApplyTransforms(ctx context.Context, dst fs.Fs, s string) string { + ci := fs.GetConfig(ctx) + if !ci.NoUnicodeNormalization { + s = norm.NFC.String(s) + } + if ci.IgnoreCaseSync || dst.Features().CaseInsensitive { + s = strings.ToLower(s) + } + return s +} + // modifyListing will modify the listing based on the results of the sync func (b *bisyncRun) modifyListing(ctx context.Context, src fs.Fs, dst fs.Fs, results []Results, queues queues, is1to2 bool) (err error) { queue := queues.copy2to1 @@ -429,6 +442,10 @@ func (b *bisyncRun) modifyListing(ctx context.Context, src fs.Fs, dst fs.Fs, res srcList.hash = src.Hashes().GetOne() dstList.hash = dst.Hashes().GetOne() } + dstListNew, err := b.loadListing(dstListing + "-new") + if err != nil { + return fmt.Errorf("cannot read new listing: %w", err) + } srcWinners := newFileList() dstWinners := newFileList() @@ -466,20 +483,47 @@ func (b *bisyncRun) modifyListing(ctx context.Context, src fs.Fs, dst fs.Fs, res } updateLists := func(side string, winners, list *fileList) { - // removals from side - for _, oldFile := range queue.ToList() { - if !winners.has(oldFile) && list.has(oldFile) && !errors.has(oldFile) { - list.remove(oldFile) - fs.Debugf(nil, "decision: removed from %s: %v", side, oldFile) - } else if winners.has(oldFile) { + for _, queueFile := range queue.ToList() { + if !winners.has(queueFile) && list.has(queueFile) && !errors.has(queueFile) { + // removals from side + list.remove(queueFile) + fs.Debugf(nil, "decision: removed from %s: %v", side, queueFile) + } else if winners.has(queueFile) { // copies to side - new := winners.get(oldFile) - list.put(oldFile, new.size, new.time, new.hash, new.id, new.flags) - fs.Debugf(nil, "decision: copied to %s: %v", side, oldFile) + new := winners.get(queueFile) + + // handle normalization according to settings + ci := fs.GetConfig(ctx) + if side == "dst" && (!ci.NoUnicodeNormalization || ci.IgnoreCaseSync || dst.Features().CaseInsensitive) { + // search list for existing file that matches queueFile when normalized + normalizedName := ApplyTransforms(ctx, dst, queueFile) + matchFound := false + matchedName := "" + for _, filename := range dstListNew.list { + if ApplyTransforms(ctx, dst, filename) == normalizedName { + matchFound = true + matchedName = filename // original, not normalized + break + } + } + if matchFound && matchedName != queueFile { + // use the (non-identical) existing name, unless --fix-case + if ci.FixCase { + fs.Debugf(direction, "removing %s and adding %s as --fix-case was specified", matchedName, queueFile) + list.remove(matchedName) + } else { + fs.Debugf(direction, "casing/unicode difference detected. using %s instead of %s", matchedName, queueFile) + queueFile = matchedName + } + } + } + + list.put(queueFile, new.size, new.time, new.hash, new.id, new.flags) + fs.Debugf(nil, "decision: copied to %s: %v", side, queueFile) } else { - fs.Debugf(oldFile, "file in queue but missing from %s transfers", side) - if err := filterRecheck.AddFile(oldFile); err != nil { - fs.Debugf(oldFile, "error adding file to recheck filter: %v", err) + fs.Debugf(queueFile, "file in queue but missing from %s transfers", side) + if err := filterRecheck.AddFile(queueFile); err != nil { + fs.Debugf(queueFile, "error adding file to recheck filter: %v", err) } } } diff --git a/cmd/bisync/march.go b/cmd/bisync/march.go index 229f3ed63..63536063b 100644 --- a/cmd/bisync/march.go +++ b/cmd/bisync/march.go @@ -183,7 +183,3 @@ func whichPath(isPath1 bool) string { } return s } - -// TODO: -// equality check? -// unicode stuff diff --git a/cmd/bisync/operations.go b/cmd/bisync/operations.go index 67cb5b440..fce29f4d0 100644 --- a/cmd/bisync/operations.go +++ b/cmd/bisync/operations.go @@ -428,8 +428,11 @@ func (b *bisyncRun) resync(octx, fctx context.Context) error { if len(copy2to1) > 0 { b.indent("Path2", "Path1", "Resync is doing queued copies to") + resync2to1 := bilib.ToNames(copy2to1) + altNames2to1 := bilib.Names{} + b.findAltNames(octx, b.fs1, resync2to1, b.newListing1, altNames2to1) // octx does not have extra filters! - results2to1, err = b.fastCopy(octx, b.fs2, b.fs1, bilib.ToNames(copy2to1), "resync-copy2to1") + results2to1, err = b.fastCopy(octx, b.fs2, b.fs1, resync2to1, "resync-copy2to1", altNames2to1) if err != nil { b.critical = true return err @@ -516,15 +519,27 @@ func (b *bisyncRun) checkSync(listing1, listing2 string) error { return fmt.Errorf("cannot read prior listing of Path2: %w", err) } + transformList := func(files *fileList, fs fs.Fs) *fileList { + transformed := newFileList() + for _, file := range files.list { + f := files.get(file) + transformed.put(ApplyTransforms(context.Background(), fs, file), f.size, f.time, f.hash, f.id, f.flags) + } + return transformed + } + + files1Transformed := transformList(files1, b.fs1) + files2Transformed := transformList(files2, b.fs2) + ok := true for _, file := range files1.list { - if !files2.has(file) { + if !files2.has(file) && !files2Transformed.has(ApplyTransforms(context.Background(), b.fs1, file)) { b.indent("ERROR", file, "Path1 file not found in Path2") ok = false } } for _, file := range files2.list { - if !files1.has(file) { + if !files1.has(file) && !files1Transformed.has(ApplyTransforms(context.Background(), b.fs2, file)) { b.indent("ERROR", file, "Path2 file not found in Path1") ok = false } diff --git a/cmd/bisync/queue.go b/cmd/bisync/queue.go index d670c033d..d16f06fcc 100644 --- a/cmd/bisync/queue.go +++ b/cmd/bisync/queue.go @@ -14,7 +14,6 @@ import ( "github.com/rclone/rclone/fs/filter" "github.com/rclone/rclone/fs/operations" "github.com/rclone/rclone/fs/sync" - "golang.org/x/text/unicode/norm" ) // Results represents a pair of synced files, as reported by the LoggerFn @@ -130,7 +129,7 @@ func ReadResults(results io.Reader) []Results { return slice } -func (b *bisyncRun) fastCopy(ctx context.Context, fsrc, fdst fs.Fs, files bilib.Names, queueName string) ([]Results, error) { +func (b *bisyncRun) fastCopy(ctx context.Context, fsrc, fdst fs.Fs, files bilib.Names, queueName string, altNames bilib.Names) ([]Results, error) { if err := b.saveQueue(files, queueName); err != nil { return nil, err } @@ -140,9 +139,12 @@ func (b *bisyncRun) fastCopy(ctx context.Context, fsrc, fdst fs.Fs, files bilib. if err := filterCopy.AddFile(file); err != nil { return nil, err } - // macOS - if err := filterCopy.AddFile(norm.NFD.String(file)); err != nil { - return nil, err + } + if altNames.NotEmpty() { + for _, file := range altNames.ToList() { + if err := filterCopy.AddFile(file); err != nil { + return nil, err + } } } @@ -252,3 +254,22 @@ func (b *bisyncRun) saveQueue(files bilib.Names, jobName string) error { queueFile := fmt.Sprintf("%s.%s.que", b.basePath, jobName) return files.Save(queueFile) } + +func (b *bisyncRun) findAltNames(ctx context.Context, dst fs.Fs, queue bilib.Names, newListing string, altNames bilib.Names) { + ci := fs.GetConfig(ctx) + if queue.NotEmpty() && (!ci.NoUnicodeNormalization || ci.IgnoreCaseSync || b.fs1.Features().CaseInsensitive || b.fs2.Features().CaseInsensitive) { + // search list for existing file that matches queueFile when normalized + for _, queueFile := range queue.ToList() { + normalizedName := ApplyTransforms(ctx, dst, queueFile) + candidates, err := b.loadListing(newListing) + if err != nil { + fs.Errorf(candidates, "cannot read new listing: %v", err) + } + for _, filename := range candidates.list { + if ApplyTransforms(ctx, dst, filename) == normalizedName && filename != queueFile { + altNames.Add(filename) // original, not normalized + } + } + } + } +} diff --git a/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst b/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst index 7471a27cf..f0208cb9a 100644 --- a/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst +++ b/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst @@ -1,10 +1,9 @@ # bisync listing v1 from test -- 6148 md5:23b446fda9938c607142c5133cf90689 - 2000-01-01T00:00:00.000000000+0000 ".DS_Store" -- 109 md5:294d25b294ff26a5243dba914ac3fbf7 - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST" -- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt" -- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt" -- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy3.txt" -- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy4.txt" -- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy5.txt" -- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-02T00:00:00.000000000+0000 "file1.txt" -- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-02T00:00:00.000000000+0000 "subdir/file20.txt" +- 109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy3.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy4.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy5.txt" +- 19 - - 2001-01-02T00:00:00.000000000+0000 "file1.txt" +- 19 - - 2001-01-02T00:00:00.000000000+0000 "subdir/file20.txt" diff --git a/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst-new b/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst-new index f243ade04..465d2455d 100644 --- a/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst-new +++ b/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst-new @@ -1,5 +1,4 @@ # bisync listing v1 from test -- 6148 md5:23b446fda9938c607142c5133cf90689 - 2000-01-01T00:00:00.000000000+0000 ".DS_Store" - 109 md5:294d25b294ff26a5243dba914ac3fbf7 - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST" - 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt" - 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt" diff --git a/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst-old b/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst-old index 991f89b82..7819f9efc 100644 --- a/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst-old +++ b/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst-old @@ -1,10 +1,9 @@ # bisync listing v1 from test -- 6148 md5:23b446fda9938c607142c5133cf90689 - 2000-01-01T00:00:00.000000000+0000 ".DS_Store" -- 109 md5:294d25b294ff26a5243dba914ac3fbf7 - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST" -- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt" -- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt" -- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy3.txt" -- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy4.txt" -- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy5.txt" -- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.txt" -- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "subdir/file20.txt" +- 109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy3.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy4.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy5.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "subdir/file20.txt" diff --git a/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst b/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst index 7471a27cf..f0208cb9a 100644 --- a/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst +++ b/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst @@ -1,10 +1,9 @@ # bisync listing v1 from test -- 6148 md5:23b446fda9938c607142c5133cf90689 - 2000-01-01T00:00:00.000000000+0000 ".DS_Store" -- 109 md5:294d25b294ff26a5243dba914ac3fbf7 - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST" -- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt" -- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt" -- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy3.txt" -- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy4.txt" -- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy5.txt" -- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-02T00:00:00.000000000+0000 "file1.txt" -- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-02T00:00:00.000000000+0000 "subdir/file20.txt" +- 109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy3.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy4.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy5.txt" +- 19 - - 2001-01-02T00:00:00.000000000+0000 "file1.txt" +- 19 - - 2001-01-02T00:00:00.000000000+0000 "subdir/file20.txt" diff --git a/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst-new b/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst-new index dac88bd2c..9e8bc8a1a 100644 --- a/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst-new +++ b/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst-new @@ -1,5 +1,4 @@ # bisync listing v1 from test -- 6148 md5:23b446fda9938c607142c5133cf90689 - 2000-01-01T00:00:00.000000000+0000 ".DS_Store" - 109 md5:294d25b294ff26a5243dba914ac3fbf7 - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST" - 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt" - 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt" diff --git a/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst-old b/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst-old index 991f89b82..7819f9efc 100644 --- a/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst-old +++ b/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst-old @@ -1,10 +1,9 @@ # bisync listing v1 from test -- 6148 md5:23b446fda9938c607142c5133cf90689 - 2000-01-01T00:00:00.000000000+0000 ".DS_Store" -- 109 md5:294d25b294ff26a5243dba914ac3fbf7 - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST" -- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt" -- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt" -- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy3.txt" -- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy4.txt" -- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy5.txt" -- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.txt" -- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "subdir/file20.txt" +- 109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy3.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy4.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy5.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "subdir/file20.txt" diff --git a/cmd/bisync/testdata/test_check_access_filters/golden/test.log b/cmd/bisync/testdata/test_check_access_filters/golden/test.log index c12126374..9ade7ee1b 100644 --- a/cmd/bisync/testdata/test_check_access_filters/golden/test.log +++ b/cmd/bisync/testdata/test_check_access_filters/golden/test.log @@ -142,8 +142,8 @@ INFO : Path2: 1 changes: 0 new, 0 newer, 0 older, 1 deleted INFO : Checking access health ERROR : Access test failed: Path1 count 3, Path2 count 4 - RCLONE_TEST ERROR : -  Access test failed: Path1 file not found in Path2 - RCLONE_TEST -ERROR : -  Access test failed: Path2 file not found in Path1 - subdir/RCLONE_TEST ERROR : -  Access test failed: Path2 file not found in Path1 - subdirX/subdirX1/RCLONE_TEST +ERROR : -  Access test failed: Path2 file not found in Path1 - subdir/RCLONE_TEST ERROR : Bisync critical error: check file check failed ERROR : Bisync aborted. Must run --resync to recover. Bisync error: bisync aborted diff --git a/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.copy1to2.que b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.copy1to2.que new file mode 100644 index 000000000..bd7f8e7dd --- /dev/null +++ b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.copy1to2.que @@ -0,0 +1,3 @@ +"folder/HeLlO,wOrLd!.txt" +"folder/éééö.txt" +"測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt" diff --git a/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.copy2to1.que b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.copy2to1.que new file mode 100644 index 000000000..85deec9f9 --- /dev/null +++ b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.copy2to1.que @@ -0,0 +1,3 @@ +"file1.txt" +"folder/hello,WORLD!.txt" +"folder/éééö.txt" diff --git a/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path1.lst b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path1.lst new file mode 100644 index 000000000..0cdecd335 --- /dev/null +++ b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path1.lst @@ -0,0 +1,5 @@ +# bisync listing v1 from test +- 19 - - 2001-01-05T00:00:00.000000000+0000 "file1.txt" +- 19 - - 2001-01-05T00:00:00.000000000+0000 "folder/HeLlO,wOrLd!.txt" +- 19 - - 2001-01-05T00:00:00.000000000+0000 "folder/éééö.txt" +- 19 - - 2001-01-05T00:00:00.000000000+0000 "測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt" diff --git a/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path1.lst-new b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path1.lst-new new file mode 100644 index 000000000..bd44aea5b --- /dev/null +++ b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path1.lst-new @@ -0,0 +1,5 @@ +# bisync listing v1 from test +- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-03T00:00:00.000000000+0000 "file1.txt" +- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-05T00:00:00.000000000+0000 "folder/HeLlO,wOrLd!.txt" +- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-05T00:00:00.000000000+0000 "folder/éééö.txt" +- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-05T00:00:00.000000000+0000 "測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt" diff --git a/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path1.lst-old b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path1.lst-old new file mode 100644 index 000000000..441434161 --- /dev/null +++ b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path1.lst-old @@ -0,0 +1,5 @@ +# bisync listing v1 from test +- 19 - - 2001-01-03T00:00:00.000000000+0000 "file1.txt" +- 19 - - 2001-01-03T00:00:00.000000000+0000 "folder/HeLlO,wOrLd!.txt" +- 19 - - 2001-01-03T00:00:00.000000000+0000 "folder/éééö.txt" +- 19 - - 2001-01-02T00:00:00.000000000+0000 "測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt" diff --git a/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst new file mode 100644 index 000000000..f7c2c0ea6 --- /dev/null +++ b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst @@ -0,0 +1,5 @@ +# bisync listing v1 from test +- 19 - - 2001-01-05T00:00:00.000000000+0000 "file1.txt" +- 19 - - 2001-01-05T00:00:00.000000000+0000 "folder/hello,WORLD!.txt" +- 19 - - 2001-01-05T00:00:00.000000000+0000 "folder/éééö.txt" +- 19 - - 2001-01-05T00:00:00.000000000+0000 "測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt" diff --git a/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst-new b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst-new new file mode 100644 index 000000000..225fe66c1 --- /dev/null +++ b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst-new @@ -0,0 +1,5 @@ +# bisync listing v1 from test +- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-05T00:00:00.000000000+0000 "file1.txt" +- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-05T00:00:00.000000000+0000 "folder/hello,WORLD!.txt" +- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-05T00:00:00.000000000+0000 "folder/éééö.txt" +- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-02T00:00:00.000000000+0000 "測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt" diff --git a/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst-old b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst-old new file mode 100644 index 000000000..ab1695601 --- /dev/null +++ b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst-old @@ -0,0 +1,5 @@ +# bisync listing v1 from test +- 19 - - 2001-01-03T00:00:00.000000000+0000 "file1.txt" +- 19 - - 2001-01-03T00:00:00.000000000+0000 "folder/hello,WORLD!.txt" +- 19 - - 2001-01-03T00:00:00.000000000+0000 "folder/éééö.txt" +- 19 - - 2001-01-02T00:00:00.000000000+0000 "測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt" diff --git a/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.resync-copy2to1.que b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.resync-copy2to1.que new file mode 100644 index 000000000..007f0aada --- /dev/null +++ b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.resync-copy2to1.que @@ -0,0 +1,2 @@ +"folder/hello,WORLD!.txt" +"folder/éééö.txt" diff --git a/cmd/bisync/testdata/test_normalization/golden/test.log b/cmd/bisync/testdata/test_normalization/golden/test.log new file mode 100644 index 000000000..accd69d14 --- /dev/null +++ b/cmd/bisync/testdata/test_normalization/golden/test.log @@ -0,0 +1,144 @@ +(01) : test normalization + + +(02) : touch-copy 2001-01-02 {datadir/}file1.txt {path2/} +(03) : test initial bisync +(04) : bisync resync +INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Copying unique Path2 files to Path1 +INFO : Resynching Path1 to Path2 +INFO : Resync updating listings +INFO : Bisync successful + + +(05) : copy-as-NFC {datadir/}file1.txt {path1/}測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éééö 測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éééö.txt +(06) : copy-as-NFC {datadir/}file1.txt {path1/}folder éééö.txt +(07) : copy-as-NFC {datadir/}file1.txt {path1/}folder HeLlO,wOrLd!.txt + + +(08) : touch-copy 2001-01-03 {datadir/}file1.txt {path2/} +(09) : copy-as-NFD {datadir/}file1.txt {path2/}folder éééö.txt +(10) : copy-as-NFD {datadir/}file1.txt {path2/}folder hello,WORLD!.txt + +(11) : test bisync run with fix-case +(12) : bisync fix-case +INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Building Path1 and Path2 listings +INFO : Path1 checking for diffs +INFO : - Path1 File is new - folder/HeLlO,wOrLd!.txt +INFO : - Path1 File is new - folder/éééö.txt +INFO : - Path1 File is new - "測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt" +INFO : Path1: 3 changes: 3 new, 0 newer, 0 older, 0 deleted +INFO : Path2 checking for diffs +INFO : - Path2 File is newer - file1.txt +INFO : - Path2 File is new - folder/éééö.txt +INFO : - Path2 File is new - folder/hello,WORLD!.txt +INFO : Path2: 3 changes: 2 new, 1 newer, 0 older, 0 deleted +INFO : Applying changes +INFO : - Path1 Queue copy to Path2 - {path2/}folder/HeLlO,wOrLd!.txt +INFO : - Path1 Queue copy to Path2 - {path2/}folder/éééö.txt +INFO : - Path1 Queue copy to Path2 - "{path2/}測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt" +INFO : - Path2 Queue copy to Path1 - {path1/}file1.txt +INFO : - Path2 Queue copy to Path1 - {path1/}folder/éééö.txt +INFO : - Path2 Queue copy to Path1 - {path1/}folder/hello,WORLD!.txt +INFO : - Path2 Do queued copies to - Path1 +INFO : folder/HeLlO,wOrLd!.txt: Fixed case by renaming to: folder/hello,WORLD!.txt +INFO : folder/éééö.txt: Fixed case by renaming to: folder/éééö.txt +INFO : - Path1 Do queued copies to - Path2 +INFO : Updating listings +INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" +INFO : Bisync successful + + +(13) : purge-children {path1/} +(14) : purge-children {path2/} +(15) : touch-copy 2001-01-02 {datadir/}file1.txt {path2/} +(16) : bisync resync +INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Copying unique Path2 files to Path1 +INFO : - Path2 Resync will copy to Path1 - file1.txt +INFO : - Path2 Resync is doing queued copies to - Path1 +INFO : Resynching Path1 to Path2 +INFO : Resync updating listings +INFO : Bisync successful + + +(17) : copy-as-NFC {datadir/}file1.txt {path1/}測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éééö 測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éééö.txt +(18) : copy-as-NFC {datadir/}file1.txt {path1/}folder éééö.txt +(19) : copy-as-NFC {datadir/}file1.txt {path1/}folder HeLlO,wOrLd!.txt + + +(20) : touch-copy 2001-01-03 {datadir/}file1.txt {path2/} +(21) : copy-as-NFD {datadir/}file1.txt {path2/}folder éééö.txt +(22) : copy-as-NFD {datadir/}file1.txt {path2/}folder hello,WORLD!.txt + +(23) : test bisync run with normalization +(24) : bisync norm force +INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Building Path1 and Path2 listings +INFO : Path1 checking for diffs +INFO : - Path1 File is new - folder/HeLlO,wOrLd!.txt +INFO : - Path1 File is new - folder/éééö.txt +INFO : - Path1 File is new - "測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt" +INFO : Path1: 3 changes: 3 new, 0 newer, 0 older, 0 deleted +INFO : Path2 checking for diffs +INFO : - Path2 File is newer - file1.txt +INFO : - Path2 File is new - folder/éééö.txt +INFO : - Path2 File is new - folder/hello,WORLD!.txt +INFO : Path2: 3 changes: 2 new, 1 newer, 0 older, 0 deleted +INFO : Applying changes +INFO : - Path1 Queue copy to Path2 - {path2/}folder/HeLlO,wOrLd!.txt +INFO : - Path1 Queue copy to Path2 - {path2/}folder/éééö.txt +INFO : - Path1 Queue copy to Path2 - "{path2/}測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt" +INFO : - Path2 Queue copy to Path1 - {path1/}file1.txt +INFO : - Path2 Queue copy to Path1 - {path1/}folder/éééö.txt +INFO : - Path2 Queue copy to Path1 - {path1/}folder/hello,WORLD!.txt +INFO : - Path2 Do queued copies to - Path1 +INFO : - Path1 Do queued copies to - Path2 +INFO : Updating listings +INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" +INFO : Bisync successful + +(25) : test resync +(26) : bisync resync norm +INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Copying unique Path2 files to Path1 +INFO : - Path2 Resync will copy to Path1 - folder/éééö.txt +INFO : - Path2 Resync will copy to Path1 - folder/hello,WORLD!.txt +INFO : - Path2 Resync is doing queued copies to - Path1 +INFO : Resynching Path1 to Path2 +INFO : Resync updating listings +INFO : Bisync successful + +(27) : test changed on both paths +(28) : touch-copy 2001-01-05 {datadir/}file1.txt {path2/} +(29) : copy-as-NFC {datadir/}file1.txt {path1/}測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éééö 測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éééö.txt +(30) : copy-as-NFC {datadir/}file1.txt {path1/}folder éééö.txt +(31) : copy-as-NFC {datadir/}file1.txt {path1/}folder HeLlO,wOrLd!.txt +(32) : copy-as-NFD {datadir/}file1.txt {path2/}folder éééö.txt +(33) : copy-as-NFD {datadir/}file1.txt {path2/}folder hello,WORLD!.txt +(34) : bisync norm +INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Building Path1 and Path2 listings +INFO : Path1 checking for diffs +INFO : - Path1 File is newer - folder/HeLlO,wOrLd!.txt +INFO : - Path1 File is newer - folder/éééö.txt +INFO : - Path1 File is newer - "測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt" +INFO : Path1: 3 changes: 0 new, 3 newer, 0 older, 0 deleted +INFO : Path2 checking for diffs +INFO : - Path2 File is newer - file1.txt +INFO : - Path2 File is newer - folder/éééö.txt +INFO : - Path2 File is newer - folder/hello,WORLD!.txt +INFO : Path2: 3 changes: 0 new, 3 newer, 0 older, 0 deleted +INFO : Applying changes +INFO : - Path1 Queue copy to Path2 - {path2/}folder/HeLlO,wOrLd!.txt +INFO : - Path1 Queue copy to Path2 - {path2/}folder/éééö.txt +INFO : - Path1 Queue copy to Path2 - "{path2/}測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt" +INFO : - Path2 Queue copy to Path1 - {path1/}file1.txt +INFO : - Path2 Queue copy to Path1 - {path1/}folder/éééö.txt +INFO : - Path2 Queue copy to Path1 - {path1/}folder/hello,WORLD!.txt +INFO : - Path2 Do queued copies to - Path1 +INFO : - Path1 Do queued copies to - Path2 +INFO : Updating listings +INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" +INFO : Bisync successful diff --git a/cmd/bisync/testdata/test_normalization/initial/RCLONE_TEST b/cmd/bisync/testdata/test_normalization/initial/RCLONE_TEST new file mode 100644 index 000000000..d8ca97c2a --- /dev/null +++ b/cmd/bisync/testdata/test_normalization/initial/RCLONE_TEST @@ -0,0 +1 @@ +This file is used for testing the health of rclone accesses to the local/remote file system. Do not delete. diff --git a/cmd/bisync/testdata/test_normalization/initial/file1.txt b/cmd/bisync/testdata/test_normalization/initial/file1.txt new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/bisync/testdata/test_normalization/initial/測試_Русский_ _ _ě_áñ/filename_contains_ě_.txt b/cmd/bisync/testdata/test_normalization/initial/測試_Русский_ _ _ě_áñ/filename_contains_ě_.txt new file mode 100644 index 000000000..a378eaef5 --- /dev/null +++ b/cmd/bisync/testdata/test_normalization/initial/測試_Русский_ _ _ě_áñ/filename_contains_ě_.txt @@ -0,0 +1,7 @@ +maana << Coded as xF1, which is the unicode ID name, but not the correct byte stream coding. +filename_contains_\u011b_ +filename_contains_e_ + +_mañana_funcionará.txt + +file_enconde_mañana_funcionará << Valid byte stream unicode is read correctly diff --git a/cmd/bisync/testdata/test_normalization/initial/測試_Русский_ _ _ě_áñ/filename_contains_ࢺ_.txt b/cmd/bisync/testdata/test_normalization/initial/測試_Русский_ _ _ě_áñ/filename_contains_ࢺ_.txt new file mode 100644 index 000000000..a378eaef5 --- /dev/null +++ b/cmd/bisync/testdata/test_normalization/initial/測試_Русский_ _ _ě_áñ/filename_contains_ࢺ_.txt @@ -0,0 +1,7 @@ +maana << Coded as xF1, which is the unicode ID name, but not the correct byte stream coding. +filename_contains_\u011b_ +filename_contains_e_ + +_mañana_funcionará.txt + +file_enconde_mañana_funcionará << Valid byte stream unicode is read correctly diff --git a/cmd/bisync/testdata/test_normalization/initial/測試_Русский_ _ _ě_áñ/測試_check file b/cmd/bisync/testdata/test_normalization/initial/測試_Русский_ _ _ě_áñ/測試_check file new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/bisync/testdata/test_normalization/modfiles/file1.txt b/cmd/bisync/testdata/test_normalization/modfiles/file1.txt new file mode 100644 index 000000000..464147f09 --- /dev/null +++ b/cmd/bisync/testdata/test_normalization/modfiles/file1.txt @@ -0,0 +1 @@ +This file is newer diff --git a/cmd/bisync/testdata/test_normalization/modfiles/測試_filtersfile.txt b/cmd/bisync/testdata/test_normalization/modfiles/測試_filtersfile.txt new file mode 100644 index 000000000..c85f097fc --- /dev/null +++ b/cmd/bisync/testdata/test_normalization/modfiles/測試_filtersfile.txt @@ -0,0 +1,5 @@ +# Test filters file +# Note that this test checks for bisync's access to and usage of a filters file, not an extensive test of rclone's filter capability + +# Exclude fileZ.txt in root only. The copy in the subdir should be found and synched. +- /fileZ.txt diff --git a/cmd/bisync/testdata/test_normalization/scenario.txt b/cmd/bisync/testdata/test_normalization/scenario.txt new file mode 100644 index 000000000..e44265fd2 --- /dev/null +++ b/cmd/bisync/testdata/test_normalization/scenario.txt @@ -0,0 +1,53 @@ +test normalization +# Tests support for --no-unicode-normalization and --ignore-case-sync +# note: this test is written carefully to be runnable regardless of case/unicode sensitivity +# i.e. the results should be the same on linux and macOS + +# force specific modification time since file time is lost through git +touch-copy 2001-01-02 {datadir/}file1.txt {path2/} +test initial bisync +bisync resync + +# copy NFC version to Path1 +copy-as-NFC {datadir/}file1.txt {path1/}測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éééö 測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éééö.txt +copy-as-NFC {datadir/}file1.txt {path1/}folder éééö.txt +copy-as-NFC {datadir/}file1.txt {path1/}folder HeLlO,wOrLd!.txt + +# place newer NFD version on Path2 +touch-copy 2001-01-03 {datadir/}file1.txt {path2/} +copy-as-NFD {datadir/}file1.txt {path2/}folder éééö.txt +copy-as-NFD {datadir/}file1.txt {path2/}folder hello,WORLD!.txt + +test bisync run with fix-case +bisync fix-case + +# purge and reset +purge-children {path1/} +purge-children {path2/} +touch-copy 2001-01-02 {datadir/}file1.txt {path2/} +bisync resync + +# copy NFC version to Path1 +copy-as-NFC {datadir/}file1.txt {path1/}測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éééö 測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éééö.txt +copy-as-NFC {datadir/}file1.txt {path1/}folder éééö.txt +copy-as-NFC {datadir/}file1.txt {path1/}folder HeLlO,wOrLd!.txt + +# place newer NFD version on Path2 +touch-copy 2001-01-03 {datadir/}file1.txt {path2/} +copy-as-NFD {datadir/}file1.txt {path2/}folder éééö.txt +copy-as-NFD {datadir/}file1.txt {path2/}folder hello,WORLD!.txt + +test bisync run with normalization +bisync norm force + +test resync +bisync resync norm + +test changed on both paths +touch-copy 2001-01-05 {datadir/}file1.txt {path2/} +copy-as-NFC {datadir/}file1.txt {path1/}測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éééö 測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éééö.txt +copy-as-NFC {datadir/}file1.txt {path1/}folder éééö.txt +copy-as-NFC {datadir/}file1.txt {path1/}folder HeLlO,wOrLd!.txt +copy-as-NFD {datadir/}file1.txt {path2/}folder éééö.txt +copy-as-NFD {datadir/}file1.txt {path2/}folder hello,WORLD!.txt +bisync norm \ No newline at end of file diff --git a/docs/content/bisync.md b/docs/content/bisync.md index b01e82cc0..109010827 100644 --- a/docs/content/bisync.md +++ b/docs/content/bisync.md @@ -588,13 +588,26 @@ and refuses to run again until the user runs a `--resync` (unless using `--resil The best workaround at the moment is to set any backend-specific flags in the [config file](/commands/rclone_config/) instead of specifying them with command flags. (You can still override them as needed for other rclone commands.) -### Case sensitivity +### Case (and unicode) sensitivity {#case-sensitivity} Synching with **case-insensitive** filesystems, such as Windows or `Box`, -can result in file name conflicts. This will be fixed in a future release. -The near-term workaround is to make sure that files on both sides +can result in unusual behavior. As of `v1.65`, case and unicode form differences no longer cause critical errors, +however they may cause unexpected delta outcomes, due to the delta engine still being case-sensitive. +This will be fixed in a future release. The near-term workaround is to make sure that files on both sides don't have spelling case differences (`Smile.jpg` vs. `smile.jpg`). +The same limitation applies to Unicode normalization forms. +This [particularly applies to macOS](https://github.com/rclone/rclone/issues/7270), +which prefers NFD and sometimes auto-converts filenames from the NFC form used by most other platforms. +This should no longer cause bisync to fail entirely, but may cause surprising delta results, as explained above. + +See the following options (all of which are supported by bisync) to control this behavior more granularly: +- [`--fix-case`](/docs/#fix-case) +- [`--ignore-case-sync`](/docs/#ignore-case-sync) +- [`--no-unicode-normalization`](/docs/#no-unicode-normalization) +- [`--local-unicode-normalization`](/local/#local-unicode-normalization) and +[`--local-case-sensitive`](/local/#local-case-sensitive) (caution: these are normally not what you want.) + ## Windows support {#windows} Bisync has been tested on Windows 8.1, Windows 10 Pro 64-bit and on Windows @@ -1278,6 +1291,7 @@ about _Unison_ and synchronization in general. * A few basic terminal colors are now supported, controllable with [`--color`](/docs/#color-when) (`AUTO`|`NEVER`|`ALWAYS`) * Initial listing snapshots of Path1 and Path2 are now generated concurrently, using the same "march" infrastructure as `check` and `sync`, for performance improvements and less [risk of error](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=4.%20Listings%20should%20alternate%20between%20paths%20to%20minimize%20errors). +* Better handling of unicode normalization and case insensitivity, support for [`--fix-case`](/docs/#fix-case), [`--ignore-case-sync`](/docs/#ignore-case-sync), [`--no-unicode-normalization`](/docs/#no-unicode-normalization) ### `v1.64` * Fixed an [issue](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=1.%20Dry%20runs%20are%20not%20completely%20dry)