diff --git a/cmd/bisync/bisync_test.go b/cmd/bisync/bisync_test.go index 11a0862c3..185981a2c 100644 --- a/cmd/bisync/bisync_test.go +++ b/cmd/bisync/bisync_test.go @@ -98,10 +98,13 @@ var logHoppers = []string{ // subdirectories. The order inconsistency initially showed up in the // listings and triggered reordering of log messages, but the actual // files will in fact match. - `ERROR : - +Access test failed: Path[12] file not found in Path[12] - .*`, + `.* +.....Access test failed: Path[12] file not found in Path[12].*`, // Test case `resync` suffered from the order of queued copies. `(?:INFO |NOTICE): - Path2 Resync will copy to Path1 +- .*`, + + // Test case `normalization` can have random order of fix-case files. + `(?:INFO |NOTICE): .*: Fixed case by renaming to: .*`, } // Some log lines can contain Windows path separator that must be @@ -546,6 +549,16 @@ func (b *bisyncTest) runTestStep(ctx context.Context, line string) (err error) { case "copy-as": b.checkArgs(args, 3, 3) return b.copyFile(ctx, args[1], args[2], args[3]) + case "copy-as-NFC": + b.checkArgs(args, 3, 3) + ci.NoUnicodeNormalization = true + ci.FixCase = true + return b.copyFile(ctx, args[1], norm.NFC.String(args[2]), norm.NFC.String(args[3])) + case "copy-as-NFD": + b.checkArgs(args, 3, 3) + ci.NoUnicodeNormalization = true + ci.FixCase = true + return b.copyFile(ctx, args[1], norm.NFD.String(args[2]), norm.NFD.String(args[3])) case "copy-dir", "sync-dir": b.checkArgs(args, 2, 2) if fsrc, err = cache.Get(ctx, args[1]); err != nil { @@ -565,6 +578,9 @@ func (b *bisyncTest) runTestStep(ctx context.Context, line string) (err error) { b.checkArgs(args, 1, 1) return b.listSubdirs(ctx, args[1]) case "bisync": + ci.NoUnicodeNormalization = false + ci.IgnoreCaseSync = false + // ci.FixCase = true return b.runBisync(ctx, args[1:]) case "test-func": b.TestFn = testFunc @@ -668,6 +684,16 @@ func (b *bisyncTest) runBisync(ctx context.Context, args []string) (err error) { fs2 = addSubdir(b.path2, val) case "ignore-listing-checksum": opt.IgnoreListingChecksum = true + case "no-norm": + ci.NoUnicodeNormalization = true + ci.IgnoreCaseSync = false + case "norm": + ci.NoUnicodeNormalization = false + ci.IgnoreCaseSync = true + case "fix-case": + ci.NoUnicodeNormalization = false + ci.IgnoreCaseSync = true + ci.FixCase = true default: return fmt.Errorf("invalid bisync option %q", arg) } diff --git a/cmd/bisync/deltas.go b/cmd/bisync/deltas.go index 3167dfb00..d0c0cd8aa 100644 --- a/cmd/bisync/deltas.go +++ b/cmd/bisync/deltas.go @@ -381,11 +381,17 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change } } + // find alternate names according to normalization settings + altNames1to2 := bilib.Names{} + altNames2to1 := bilib.Names{} + b.findAltNames(ctx, b.fs1, copy2to1, b.newListing1, altNames2to1) + b.findAltNames(ctx, b.fs2, copy1to2, b.newListing2, altNames1to2) + // Do the batch operation if copy2to1.NotEmpty() { changes1 = true b.indent("Path2", "Path1", "Do queued copies to") - results2to1, err = b.fastCopy(ctx, b.fs2, b.fs1, copy2to1, "copy2to1") + results2to1, err = b.fastCopy(ctx, b.fs2, b.fs1, copy2to1, "copy2to1", altNames2to1) if err != nil { return } @@ -397,7 +403,7 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change if copy1to2.NotEmpty() { changes2 = true b.indent("Path1", "Path2", "Do queued copies to") - results1to2, err = b.fastCopy(ctx, b.fs1, b.fs2, copy1to2, "copy1to2") + results1to2, err = b.fastCopy(ctx, b.fs1, b.fs2, copy1to2, "copy1to2", altNames1to2) if err != nil { return } diff --git a/cmd/bisync/listing.go b/cmd/bisync/listing.go index 7875036fa..f94883d97 100644 --- a/cmd/bisync/listing.go +++ b/cmd/bisync/listing.go @@ -20,6 +20,7 @@ import ( "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/fs/operations" "golang.org/x/exp/slices" + "golang.org/x/text/unicode/norm" ) // ListingHeader defines first line of a listing @@ -400,6 +401,18 @@ func ConvertPrecision(Modtime time.Time, dst fs.Fs) time.Time { return Modtime } +// ApplyTransforms handles unicode and case normalization +func ApplyTransforms(ctx context.Context, dst fs.Fs, s string) string { + ci := fs.GetConfig(ctx) + if !ci.NoUnicodeNormalization { + s = norm.NFC.String(s) + } + if ci.IgnoreCaseSync || dst.Features().CaseInsensitive { + s = strings.ToLower(s) + } + return s +} + // modifyListing will modify the listing based on the results of the sync func (b *bisyncRun) modifyListing(ctx context.Context, src fs.Fs, dst fs.Fs, results []Results, queues queues, is1to2 bool) (err error) { queue := queues.copy2to1 @@ -429,6 +442,10 @@ func (b *bisyncRun) modifyListing(ctx context.Context, src fs.Fs, dst fs.Fs, res srcList.hash = src.Hashes().GetOne() dstList.hash = dst.Hashes().GetOne() } + dstListNew, err := b.loadListing(dstListing + "-new") + if err != nil { + return fmt.Errorf("cannot read new listing: %w", err) + } srcWinners := newFileList() dstWinners := newFileList() @@ -466,20 +483,47 @@ func (b *bisyncRun) modifyListing(ctx context.Context, src fs.Fs, dst fs.Fs, res } updateLists := func(side string, winners, list *fileList) { - // removals from side - for _, oldFile := range queue.ToList() { - if !winners.has(oldFile) && list.has(oldFile) && !errors.has(oldFile) { - list.remove(oldFile) - fs.Debugf(nil, "decision: removed from %s: %v", side, oldFile) - } else if winners.has(oldFile) { + for _, queueFile := range queue.ToList() { + if !winners.has(queueFile) && list.has(queueFile) && !errors.has(queueFile) { + // removals from side + list.remove(queueFile) + fs.Debugf(nil, "decision: removed from %s: %v", side, queueFile) + } else if winners.has(queueFile) { // copies to side - new := winners.get(oldFile) - list.put(oldFile, new.size, new.time, new.hash, new.id, new.flags) - fs.Debugf(nil, "decision: copied to %s: %v", side, oldFile) + new := winners.get(queueFile) + + // handle normalization according to settings + ci := fs.GetConfig(ctx) + if side == "dst" && (!ci.NoUnicodeNormalization || ci.IgnoreCaseSync || dst.Features().CaseInsensitive) { + // search list for existing file that matches queueFile when normalized + normalizedName := ApplyTransforms(ctx, dst, queueFile) + matchFound := false + matchedName := "" + for _, filename := range dstListNew.list { + if ApplyTransforms(ctx, dst, filename) == normalizedName { + matchFound = true + matchedName = filename // original, not normalized + break + } + } + if matchFound && matchedName != queueFile { + // use the (non-identical) existing name, unless --fix-case + if ci.FixCase { + fs.Debugf(direction, "removing %s and adding %s as --fix-case was specified", matchedName, queueFile) + list.remove(matchedName) + } else { + fs.Debugf(direction, "casing/unicode difference detected. using %s instead of %s", matchedName, queueFile) + queueFile = matchedName + } + } + } + + list.put(queueFile, new.size, new.time, new.hash, new.id, new.flags) + fs.Debugf(nil, "decision: copied to %s: %v", side, queueFile) } else { - fs.Debugf(oldFile, "file in queue but missing from %s transfers", side) - if err := filterRecheck.AddFile(oldFile); err != nil { - fs.Debugf(oldFile, "error adding file to recheck filter: %v", err) + fs.Debugf(queueFile, "file in queue but missing from %s transfers", side) + if err := filterRecheck.AddFile(queueFile); err != nil { + fs.Debugf(queueFile, "error adding file to recheck filter: %v", err) } } } diff --git a/cmd/bisync/march.go b/cmd/bisync/march.go index 229f3ed63..63536063b 100644 --- a/cmd/bisync/march.go +++ b/cmd/bisync/march.go @@ -183,7 +183,3 @@ func whichPath(isPath1 bool) string { } return s } - -// TODO: -// equality check? -// unicode stuff diff --git a/cmd/bisync/operations.go b/cmd/bisync/operations.go index 67cb5b440..fce29f4d0 100644 --- a/cmd/bisync/operations.go +++ b/cmd/bisync/operations.go @@ -428,8 +428,11 @@ func (b *bisyncRun) resync(octx, fctx context.Context) error { if len(copy2to1) > 0 { b.indent("Path2", "Path1", "Resync is doing queued copies to") + resync2to1 := bilib.ToNames(copy2to1) + altNames2to1 := bilib.Names{} + b.findAltNames(octx, b.fs1, resync2to1, b.newListing1, altNames2to1) // octx does not have extra filters! - results2to1, err = b.fastCopy(octx, b.fs2, b.fs1, bilib.ToNames(copy2to1), "resync-copy2to1") + results2to1, err = b.fastCopy(octx, b.fs2, b.fs1, resync2to1, "resync-copy2to1", altNames2to1) if err != nil { b.critical = true return err @@ -516,15 +519,27 @@ func (b *bisyncRun) checkSync(listing1, listing2 string) error { return fmt.Errorf("cannot read prior listing of Path2: %w", err) } + transformList := func(files *fileList, fs fs.Fs) *fileList { + transformed := newFileList() + for _, file := range files.list { + f := files.get(file) + transformed.put(ApplyTransforms(context.Background(), fs, file), f.size, f.time, f.hash, f.id, f.flags) + } + return transformed + } + + files1Transformed := transformList(files1, b.fs1) + files2Transformed := transformList(files2, b.fs2) + ok := true for _, file := range files1.list { - if !files2.has(file) { + if !files2.has(file) && !files2Transformed.has(ApplyTransforms(context.Background(), b.fs1, file)) { b.indent("ERROR", file, "Path1 file not found in Path2") ok = false } } for _, file := range files2.list { - if !files1.has(file) { + if !files1.has(file) && !files1Transformed.has(ApplyTransforms(context.Background(), b.fs2, file)) { b.indent("ERROR", file, "Path2 file not found in Path1") ok = false } diff --git a/cmd/bisync/queue.go b/cmd/bisync/queue.go index d670c033d..d16f06fcc 100644 --- a/cmd/bisync/queue.go +++ b/cmd/bisync/queue.go @@ -14,7 +14,6 @@ import ( "github.com/rclone/rclone/fs/filter" "github.com/rclone/rclone/fs/operations" "github.com/rclone/rclone/fs/sync" - "golang.org/x/text/unicode/norm" ) // Results represents a pair of synced files, as reported by the LoggerFn @@ -130,7 +129,7 @@ func ReadResults(results io.Reader) []Results { return slice } -func (b *bisyncRun) fastCopy(ctx context.Context, fsrc, fdst fs.Fs, files bilib.Names, queueName string) ([]Results, error) { +func (b *bisyncRun) fastCopy(ctx context.Context, fsrc, fdst fs.Fs, files bilib.Names, queueName string, altNames bilib.Names) ([]Results, error) { if err := b.saveQueue(files, queueName); err != nil { return nil, err } @@ -140,9 +139,12 @@ func (b *bisyncRun) fastCopy(ctx context.Context, fsrc, fdst fs.Fs, files bilib. if err := filterCopy.AddFile(file); err != nil { return nil, err } - // macOS - if err := filterCopy.AddFile(norm.NFD.String(file)); err != nil { - return nil, err + } + if altNames.NotEmpty() { + for _, file := range altNames.ToList() { + if err := filterCopy.AddFile(file); err != nil { + return nil, err + } } } @@ -252,3 +254,22 @@ func (b *bisyncRun) saveQueue(files bilib.Names, jobName string) error { queueFile := fmt.Sprintf("%s.%s.que", b.basePath, jobName) return files.Save(queueFile) } + +func (b *bisyncRun) findAltNames(ctx context.Context, dst fs.Fs, queue bilib.Names, newListing string, altNames bilib.Names) { + ci := fs.GetConfig(ctx) + if queue.NotEmpty() && (!ci.NoUnicodeNormalization || ci.IgnoreCaseSync || b.fs1.Features().CaseInsensitive || b.fs2.Features().CaseInsensitive) { + // search list for existing file that matches queueFile when normalized + for _, queueFile := range queue.ToList() { + normalizedName := ApplyTransforms(ctx, dst, queueFile) + candidates, err := b.loadListing(newListing) + if err != nil { + fs.Errorf(candidates, "cannot read new listing: %v", err) + } + for _, filename := range candidates.list { + if ApplyTransforms(ctx, dst, filename) == normalizedName && filename != queueFile { + altNames.Add(filename) // original, not normalized + } + } + } + } +} diff --git a/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst b/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst index 7471a27cf..f0208cb9a 100644 --- a/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst +++ b/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst @@ -1,10 +1,9 @@ # bisync listing v1 from test -- 6148 md5:23b446fda9938c607142c5133cf90689 - 2000-01-01T00:00:00.000000000+0000 ".DS_Store" -- 109 md5:294d25b294ff26a5243dba914ac3fbf7 - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST" -- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt" -- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt" -- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy3.txt" -- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy4.txt" -- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy5.txt" -- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-02T00:00:00.000000000+0000 "file1.txt" -- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-02T00:00:00.000000000+0000 "subdir/file20.txt" +- 109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy3.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy4.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy5.txt" +- 19 - - 2001-01-02T00:00:00.000000000+0000 "file1.txt" +- 19 - - 2001-01-02T00:00:00.000000000+0000 "subdir/file20.txt" diff --git a/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst-new b/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst-new index f243ade04..465d2455d 100644 --- a/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst-new +++ b/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst-new @@ -1,5 +1,4 @@ # bisync listing v1 from test -- 6148 md5:23b446fda9938c607142c5133cf90689 - 2000-01-01T00:00:00.000000000+0000 ".DS_Store" - 109 md5:294d25b294ff26a5243dba914ac3fbf7 - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST" - 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt" - 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt" diff --git a/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst-old b/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst-old index 991f89b82..7819f9efc 100644 --- a/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst-old +++ b/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst-old @@ -1,10 +1,9 @@ # bisync listing v1 from test -- 6148 md5:23b446fda9938c607142c5133cf90689 - 2000-01-01T00:00:00.000000000+0000 ".DS_Store" -- 109 md5:294d25b294ff26a5243dba914ac3fbf7 - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST" -- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt" -- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt" -- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy3.txt" -- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy4.txt" -- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy5.txt" -- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.txt" -- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "subdir/file20.txt" +- 109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy3.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy4.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy5.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "subdir/file20.txt" diff --git a/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst b/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst index 7471a27cf..f0208cb9a 100644 --- a/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst +++ b/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst @@ -1,10 +1,9 @@ # bisync listing v1 from test -- 6148 md5:23b446fda9938c607142c5133cf90689 - 2000-01-01T00:00:00.000000000+0000 ".DS_Store" -- 109 md5:294d25b294ff26a5243dba914ac3fbf7 - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST" -- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt" -- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt" -- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy3.txt" -- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy4.txt" -- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy5.txt" -- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-02T00:00:00.000000000+0000 "file1.txt" -- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-02T00:00:00.000000000+0000 "subdir/file20.txt" +- 109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy3.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy4.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy5.txt" +- 19 - - 2001-01-02T00:00:00.000000000+0000 "file1.txt" +- 19 - - 2001-01-02T00:00:00.000000000+0000 "subdir/file20.txt" diff --git a/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst-new b/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst-new index dac88bd2c..9e8bc8a1a 100644 --- a/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst-new +++ b/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst-new @@ -1,5 +1,4 @@ # bisync listing v1 from test -- 6148 md5:23b446fda9938c607142c5133cf90689 - 2000-01-01T00:00:00.000000000+0000 ".DS_Store" - 109 md5:294d25b294ff26a5243dba914ac3fbf7 - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST" - 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt" - 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt" diff --git a/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst-old b/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst-old index 991f89b82..7819f9efc 100644 --- a/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst-old +++ b/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst-old @@ -1,10 +1,9 @@ # bisync listing v1 from test -- 6148 md5:23b446fda9938c607142c5133cf90689 - 2000-01-01T00:00:00.000000000+0000 ".DS_Store" -- 109 md5:294d25b294ff26a5243dba914ac3fbf7 - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST" -- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt" -- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt" -- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy3.txt" -- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy4.txt" -- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy5.txt" -- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.txt" -- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "subdir/file20.txt" +- 109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy3.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy4.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy5.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "subdir/file20.txt" diff --git a/cmd/bisync/testdata/test_check_access_filters/golden/test.log b/cmd/bisync/testdata/test_check_access_filters/golden/test.log index c12126374..9ade7ee1b 100644 --- a/cmd/bisync/testdata/test_check_access_filters/golden/test.log +++ b/cmd/bisync/testdata/test_check_access_filters/golden/test.log @@ -142,8 +142,8 @@ INFO : Path2: 1 changes: 0 new, 0 newer, 0 older, 1 deleted INFO : Checking access health ERROR : Access test failed: Path1 count 3, Path2 count 4 - RCLONE_TEST ERROR : -  Access test failed: Path1 file not found in Path2 - RCLONE_TEST -ERROR : -  Access test failed: Path2 file not found in Path1 - subdir/RCLONE_TEST ERROR : -  Access test failed: Path2 file not found in Path1 - subdirX/subdirX1/RCLONE_TEST +ERROR : -  Access test failed: Path2 file not found in Path1 - subdir/RCLONE_TEST ERROR : Bisync critical error: check file check failed ERROR : Bisync aborted. Must run --resync to recover. Bisync error: bisync aborted diff --git a/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.copy1to2.que b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.copy1to2.que new file mode 100644 index 000000000..bd7f8e7dd --- /dev/null +++ b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.copy1to2.que @@ -0,0 +1,3 @@ +"folder/HeLlO,wOrLd!.txt" +"folder/éééö.txt" +"測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt" diff --git a/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.copy2to1.que b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.copy2to1.que new file mode 100644 index 000000000..85deec9f9 --- /dev/null +++ b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.copy2to1.que @@ -0,0 +1,3 @@ +"file1.txt" +"folder/hello,WORLD!.txt" +"folder/éééö.txt" diff --git a/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path1.lst b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path1.lst new file mode 100644 index 000000000..0cdecd335 --- /dev/null +++ b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path1.lst @@ -0,0 +1,5 @@ +# bisync listing v1 from test +- 19 - - 2001-01-05T00:00:00.000000000+0000 "file1.txt" +- 19 - - 2001-01-05T00:00:00.000000000+0000 "folder/HeLlO,wOrLd!.txt" +- 19 - - 2001-01-05T00:00:00.000000000+0000 "folder/éééö.txt" +- 19 - - 2001-01-05T00:00:00.000000000+0000 "測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt" diff --git a/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path1.lst-new b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path1.lst-new new file mode 100644 index 000000000..bd44aea5b --- /dev/null +++ b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path1.lst-new @@ -0,0 +1,5 @@ +# bisync listing v1 from test +- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-03T00:00:00.000000000+0000 "file1.txt" +- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-05T00:00:00.000000000+0000 "folder/HeLlO,wOrLd!.txt" +- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-05T00:00:00.000000000+0000 "folder/éééö.txt" +- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-05T00:00:00.000000000+0000 "測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt" diff --git a/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path1.lst-old b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path1.lst-old new file mode 100644 index 000000000..441434161 --- /dev/null +++ b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path1.lst-old @@ -0,0 +1,5 @@ +# bisync listing v1 from test +- 19 - - 2001-01-03T00:00:00.000000000+0000 "file1.txt" +- 19 - - 2001-01-03T00:00:00.000000000+0000 "folder/HeLlO,wOrLd!.txt" +- 19 - - 2001-01-03T00:00:00.000000000+0000 "folder/éééö.txt" +- 19 - - 2001-01-02T00:00:00.000000000+0000 "測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt" diff --git a/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst new file mode 100644 index 000000000..f7c2c0ea6 --- /dev/null +++ b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst @@ -0,0 +1,5 @@ +# bisync listing v1 from test +- 19 - - 2001-01-05T00:00:00.000000000+0000 "file1.txt" +- 19 - - 2001-01-05T00:00:00.000000000+0000 "folder/hello,WORLD!.txt" +- 19 - - 2001-01-05T00:00:00.000000000+0000 "folder/éééö.txt" +- 19 - - 2001-01-05T00:00:00.000000000+0000 "測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt" diff --git a/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst-new b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst-new new file mode 100644 index 000000000..225fe66c1 --- /dev/null +++ b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst-new @@ -0,0 +1,5 @@ +# bisync listing v1 from test +- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-05T00:00:00.000000000+0000 "file1.txt" +- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-05T00:00:00.000000000+0000 "folder/hello,WORLD!.txt" +- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-05T00:00:00.000000000+0000 "folder/éééö.txt" +- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-02T00:00:00.000000000+0000 "測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt" diff --git a/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst-old b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst-old new file mode 100644 index 000000000..ab1695601 --- /dev/null +++ b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst-old @@ -0,0 +1,5 @@ +# bisync listing v1 from test +- 19 - - 2001-01-03T00:00:00.000000000+0000 "file1.txt" +- 19 - - 2001-01-03T00:00:00.000000000+0000 "folder/hello,WORLD!.txt" +- 19 - - 2001-01-03T00:00:00.000000000+0000 "folder/éééö.txt" +- 19 - - 2001-01-02T00:00:00.000000000+0000 "測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt" diff --git a/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.resync-copy2to1.que b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.resync-copy2to1.que new file mode 100644 index 000000000..007f0aada --- /dev/null +++ b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.resync-copy2to1.que @@ -0,0 +1,2 @@ +"folder/hello,WORLD!.txt" +"folder/éééö.txt" diff --git a/cmd/bisync/testdata/test_normalization/golden/test.log b/cmd/bisync/testdata/test_normalization/golden/test.log new file mode 100644 index 000000000..accd69d14 --- /dev/null +++ b/cmd/bisync/testdata/test_normalization/golden/test.log @@ -0,0 +1,144 @@ +(01) : test normalization + + +(02) : touch-copy 2001-01-02 {datadir/}file1.txt {path2/} +(03) : test initial bisync +(04) : bisync resync +INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Copying unique Path2 files to Path1 +INFO : Resynching Path1 to Path2 +INFO : Resync updating listings +INFO : Bisync successful + + +(05) : copy-as-NFC {datadir/}file1.txt {path1/}測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éééö 測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éééö.txt +(06) : copy-as-NFC {datadir/}file1.txt {path1/}folder éééö.txt +(07) : copy-as-NFC {datadir/}file1.txt {path1/}folder HeLlO,wOrLd!.txt + + +(08) : touch-copy 2001-01-03 {datadir/}file1.txt {path2/} +(09) : copy-as-NFD {datadir/}file1.txt {path2/}folder éééö.txt +(10) : copy-as-NFD {datadir/}file1.txt {path2/}folder hello,WORLD!.txt + +(11) : test bisync run with fix-case +(12) : bisync fix-case +INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Building Path1 and Path2 listings +INFO : Path1 checking for diffs +INFO : - Path1 File is new - folder/HeLlO,wOrLd!.txt +INFO : - Path1 File is new - folder/éééö.txt +INFO : - Path1 File is new - "測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt" +INFO : Path1: 3 changes: 3 new, 0 newer, 0 older, 0 deleted +INFO : Path2 checking for diffs +INFO : - Path2 File is newer - file1.txt +INFO : - Path2 File is new - folder/éééö.txt +INFO : - Path2 File is new - folder/hello,WORLD!.txt +INFO : Path2: 3 changes: 2 new, 1 newer, 0 older, 0 deleted +INFO : Applying changes +INFO : - Path1 Queue copy to Path2 - {path2/}folder/HeLlO,wOrLd!.txt +INFO : - Path1 Queue copy to Path2 - {path2/}folder/éééö.txt +INFO : - Path1 Queue copy to Path2 - "{path2/}測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt" +INFO : - Path2 Queue copy to Path1 - {path1/}file1.txt +INFO : - Path2 Queue copy to Path1 - {path1/}folder/éééö.txt +INFO : - Path2 Queue copy to Path1 - {path1/}folder/hello,WORLD!.txt +INFO : - Path2 Do queued copies to - Path1 +INFO : folder/HeLlO,wOrLd!.txt: Fixed case by renaming to: folder/hello,WORLD!.txt +INFO : folder/éééö.txt: Fixed case by renaming to: folder/éééö.txt +INFO : - Path1 Do queued copies to - Path2 +INFO : Updating listings +INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" +INFO : Bisync successful + + +(13) : purge-children {path1/} +(14) : purge-children {path2/} +(15) : touch-copy 2001-01-02 {datadir/}file1.txt {path2/} +(16) : bisync resync +INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Copying unique Path2 files to Path1 +INFO : - Path2 Resync will copy to Path1 - file1.txt +INFO : - Path2 Resync is doing queued copies to - Path1 +INFO : Resynching Path1 to Path2 +INFO : Resync updating listings +INFO : Bisync successful + + +(17) : copy-as-NFC {datadir/}file1.txt {path1/}測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éééö 測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éééö.txt +(18) : copy-as-NFC {datadir/}file1.txt {path1/}folder éééö.txt +(19) : copy-as-NFC {datadir/}file1.txt {path1/}folder HeLlO,wOrLd!.txt + + +(20) : touch-copy 2001-01-03 {datadir/}file1.txt {path2/} +(21) : copy-as-NFD {datadir/}file1.txt {path2/}folder éééö.txt +(22) : copy-as-NFD {datadir/}file1.txt {path2/}folder hello,WORLD!.txt + +(23) : test bisync run with normalization +(24) : bisync norm force +INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Building Path1 and Path2 listings +INFO : Path1 checking for diffs +INFO : - Path1 File is new - folder/HeLlO,wOrLd!.txt +INFO : - Path1 File is new - folder/éééö.txt +INFO : - Path1 File is new - "測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt" +INFO : Path1: 3 changes: 3 new, 0 newer, 0 older, 0 deleted +INFO : Path2 checking for diffs +INFO : - Path2 File is newer - file1.txt +INFO : - Path2 File is new - folder/éééö.txt +INFO : - Path2 File is new - folder/hello,WORLD!.txt +INFO : Path2: 3 changes: 2 new, 1 newer, 0 older, 0 deleted +INFO : Applying changes +INFO : - Path1 Queue copy to Path2 - {path2/}folder/HeLlO,wOrLd!.txt +INFO : - Path1 Queue copy to Path2 - {path2/}folder/éééö.txt +INFO : - Path1 Queue copy to Path2 - "{path2/}測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt" +INFO : - Path2 Queue copy to Path1 - {path1/}file1.txt +INFO : - Path2 Queue copy to Path1 - {path1/}folder/éééö.txt +INFO : - Path2 Queue copy to Path1 - {path1/}folder/hello,WORLD!.txt +INFO : - Path2 Do queued copies to - Path1 +INFO : - Path1 Do queued copies to - Path2 +INFO : Updating listings +INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" +INFO : Bisync successful + +(25) : test resync +(26) : bisync resync norm +INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Copying unique Path2 files to Path1 +INFO : - Path2 Resync will copy to Path1 - folder/éééö.txt +INFO : - Path2 Resync will copy to Path1 - folder/hello,WORLD!.txt +INFO : - Path2 Resync is doing queued copies to - Path1 +INFO : Resynching Path1 to Path2 +INFO : Resync updating listings +INFO : Bisync successful + +(27) : test changed on both paths +(28) : touch-copy 2001-01-05 {datadir/}file1.txt {path2/} +(29) : copy-as-NFC {datadir/}file1.txt {path1/}測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éééö 測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éééö.txt +(30) : copy-as-NFC {datadir/}file1.txt {path1/}folder éééö.txt +(31) : copy-as-NFC {datadir/}file1.txt {path1/}folder HeLlO,wOrLd!.txt +(32) : copy-as-NFD {datadir/}file1.txt {path2/}folder éééö.txt +(33) : copy-as-NFD {datadir/}file1.txt {path2/}folder hello,WORLD!.txt +(34) : bisync norm +INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Building Path1 and Path2 listings +INFO : Path1 checking for diffs +INFO : - Path1 File is newer - folder/HeLlO,wOrLd!.txt +INFO : - Path1 File is newer - folder/éééö.txt +INFO : - Path1 File is newer - "測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt" +INFO : Path1: 3 changes: 0 new, 3 newer, 0 older, 0 deleted +INFO : Path2 checking for diffs +INFO : - Path2 File is newer - file1.txt +INFO : - Path2 File is newer - folder/éééö.txt +INFO : - Path2 File is newer - folder/hello,WORLD!.txt +INFO : Path2: 3 changes: 0 new, 3 newer, 0 older, 0 deleted +INFO : Applying changes +INFO : - Path1 Queue copy to Path2 - {path2/}folder/HeLlO,wOrLd!.txt +INFO : - Path1 Queue copy to Path2 - {path2/}folder/éééö.txt +INFO : - Path1 Queue copy to Path2 - "{path2/}測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt" +INFO : - Path2 Queue copy to Path1 - {path1/}file1.txt +INFO : - Path2 Queue copy to Path1 - {path1/}folder/éééö.txt +INFO : - Path2 Queue copy to Path1 - {path1/}folder/hello,WORLD!.txt +INFO : - Path2 Do queued copies to - Path1 +INFO : - Path1 Do queued copies to - Path2 +INFO : Updating listings +INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" +INFO : Bisync successful diff --git a/cmd/bisync/testdata/test_normalization/initial/RCLONE_TEST b/cmd/bisync/testdata/test_normalization/initial/RCLONE_TEST new file mode 100644 index 000000000..d8ca97c2a --- /dev/null +++ b/cmd/bisync/testdata/test_normalization/initial/RCLONE_TEST @@ -0,0 +1 @@ +This file is used for testing the health of rclone accesses to the local/remote file system. Do not delete. diff --git a/cmd/bisync/testdata/test_normalization/initial/file1.txt b/cmd/bisync/testdata/test_normalization/initial/file1.txt new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/bisync/testdata/test_normalization/initial/測試_Русский_ _ _ě_áñ/filename_contains_ě_.txt b/cmd/bisync/testdata/test_normalization/initial/測試_Русский_ _ _ě_áñ/filename_contains_ě_.txt new file mode 100644 index 000000000..a378eaef5 --- /dev/null +++ b/cmd/bisync/testdata/test_normalization/initial/測試_Русский_ _ _ě_áñ/filename_contains_ě_.txt @@ -0,0 +1,7 @@ +maana << Coded as xF1, which is the unicode ID name, but not the correct byte stream coding. +filename_contains_\u011b_ +filename_contains_e_ + +_mañana_funcionará.txt + +file_enconde_mañana_funcionará << Valid byte stream unicode is read correctly diff --git a/cmd/bisync/testdata/test_normalization/initial/測試_Русский_ _ _ě_áñ/filename_contains_ࢺ_.txt b/cmd/bisync/testdata/test_normalization/initial/測試_Русский_ _ _ě_áñ/filename_contains_ࢺ_.txt new file mode 100644 index 000000000..a378eaef5 --- /dev/null +++ b/cmd/bisync/testdata/test_normalization/initial/測試_Русский_ _ _ě_áñ/filename_contains_ࢺ_.txt @@ -0,0 +1,7 @@ +maana << Coded as xF1, which is the unicode ID name, but not the correct byte stream coding. +filename_contains_\u011b_ +filename_contains_e_ + +_mañana_funcionará.txt + +file_enconde_mañana_funcionará << Valid byte stream unicode is read correctly diff --git a/cmd/bisync/testdata/test_normalization/initial/測試_Русский_ _ _ě_áñ/測試_check file b/cmd/bisync/testdata/test_normalization/initial/測試_Русский_ _ _ě_áñ/測試_check file new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/bisync/testdata/test_normalization/modfiles/file1.txt b/cmd/bisync/testdata/test_normalization/modfiles/file1.txt new file mode 100644 index 000000000..464147f09 --- /dev/null +++ b/cmd/bisync/testdata/test_normalization/modfiles/file1.txt @@ -0,0 +1 @@ +This file is newer diff --git a/cmd/bisync/testdata/test_normalization/modfiles/測試_filtersfile.txt b/cmd/bisync/testdata/test_normalization/modfiles/測試_filtersfile.txt new file mode 100644 index 000000000..c85f097fc --- /dev/null +++ b/cmd/bisync/testdata/test_normalization/modfiles/測試_filtersfile.txt @@ -0,0 +1,5 @@ +# Test filters file +# Note that this test checks for bisync's access to and usage of a filters file, not an extensive test of rclone's filter capability + +# Exclude fileZ.txt in root only. The copy in the subdir should be found and synched. +- /fileZ.txt diff --git a/cmd/bisync/testdata/test_normalization/scenario.txt b/cmd/bisync/testdata/test_normalization/scenario.txt new file mode 100644 index 000000000..e44265fd2 --- /dev/null +++ b/cmd/bisync/testdata/test_normalization/scenario.txt @@ -0,0 +1,53 @@ +test normalization +# Tests support for --no-unicode-normalization and --ignore-case-sync +# note: this test is written carefully to be runnable regardless of case/unicode sensitivity +# i.e. the results should be the same on linux and macOS + +# force specific modification time since file time is lost through git +touch-copy 2001-01-02 {datadir/}file1.txt {path2/} +test initial bisync +bisync resync + +# copy NFC version to Path1 +copy-as-NFC {datadir/}file1.txt {path1/}測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éééö 測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éééö.txt +copy-as-NFC {datadir/}file1.txt {path1/}folder éééö.txt +copy-as-NFC {datadir/}file1.txt {path1/}folder HeLlO,wOrLd!.txt + +# place newer NFD version on Path2 +touch-copy 2001-01-03 {datadir/}file1.txt {path2/} +copy-as-NFD {datadir/}file1.txt {path2/}folder éééö.txt +copy-as-NFD {datadir/}file1.txt {path2/}folder hello,WORLD!.txt + +test bisync run with fix-case +bisync fix-case + +# purge and reset +purge-children {path1/} +purge-children {path2/} +touch-copy 2001-01-02 {datadir/}file1.txt {path2/} +bisync resync + +# copy NFC version to Path1 +copy-as-NFC {datadir/}file1.txt {path1/}測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éééö 測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éééö.txt +copy-as-NFC {datadir/}file1.txt {path1/}folder éééö.txt +copy-as-NFC {datadir/}file1.txt {path1/}folder HeLlO,wOrLd!.txt + +# place newer NFD version on Path2 +touch-copy 2001-01-03 {datadir/}file1.txt {path2/} +copy-as-NFD {datadir/}file1.txt {path2/}folder éééö.txt +copy-as-NFD {datadir/}file1.txt {path2/}folder hello,WORLD!.txt + +test bisync run with normalization +bisync norm force + +test resync +bisync resync norm + +test changed on both paths +touch-copy 2001-01-05 {datadir/}file1.txt {path2/} +copy-as-NFC {datadir/}file1.txt {path1/}測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éééö 測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éééö.txt +copy-as-NFC {datadir/}file1.txt {path1/}folder éééö.txt +copy-as-NFC {datadir/}file1.txt {path1/}folder HeLlO,wOrLd!.txt +copy-as-NFD {datadir/}file1.txt {path2/}folder éééö.txt +copy-as-NFD {datadir/}file1.txt {path2/}folder hello,WORLD!.txt +bisync norm \ No newline at end of file diff --git a/docs/content/bisync.md b/docs/content/bisync.md index b01e82cc0..109010827 100644 --- a/docs/content/bisync.md +++ b/docs/content/bisync.md @@ -588,13 +588,26 @@ and refuses to run again until the user runs a `--resync` (unless using `--resil The best workaround at the moment is to set any backend-specific flags in the [config file](/commands/rclone_config/) instead of specifying them with command flags. (You can still override them as needed for other rclone commands.) -### Case sensitivity +### Case (and unicode) sensitivity {#case-sensitivity} Synching with **case-insensitive** filesystems, such as Windows or `Box`, -can result in file name conflicts. This will be fixed in a future release. -The near-term workaround is to make sure that files on both sides +can result in unusual behavior. As of `v1.65`, case and unicode form differences no longer cause critical errors, +however they may cause unexpected delta outcomes, due to the delta engine still being case-sensitive. +This will be fixed in a future release. The near-term workaround is to make sure that files on both sides don't have spelling case differences (`Smile.jpg` vs. `smile.jpg`). +The same limitation applies to Unicode normalization forms. +This [particularly applies to macOS](https://github.com/rclone/rclone/issues/7270), +which prefers NFD and sometimes auto-converts filenames from the NFC form used by most other platforms. +This should no longer cause bisync to fail entirely, but may cause surprising delta results, as explained above. + +See the following options (all of which are supported by bisync) to control this behavior more granularly: +- [`--fix-case`](/docs/#fix-case) +- [`--ignore-case-sync`](/docs/#ignore-case-sync) +- [`--no-unicode-normalization`](/docs/#no-unicode-normalization) +- [`--local-unicode-normalization`](/local/#local-unicode-normalization) and +[`--local-case-sensitive`](/local/#local-case-sensitive) (caution: these are normally not what you want.) + ## Windows support {#windows} Bisync has been tested on Windows 8.1, Windows 10 Pro 64-bit and on Windows @@ -1278,6 +1291,7 @@ about _Unison_ and synchronization in general. * A few basic terminal colors are now supported, controllable with [`--color`](/docs/#color-when) (`AUTO`|`NEVER`|`ALWAYS`) * Initial listing snapshots of Path1 and Path2 are now generated concurrently, using the same "march" infrastructure as `check` and `sync`, for performance improvements and less [risk of error](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=4.%20Listings%20should%20alternate%20between%20paths%20to%20minimize%20errors). +* Better handling of unicode normalization and case insensitivity, support for [`--fix-case`](/docs/#fix-case), [`--ignore-case-sync`](/docs/#ignore-case-sync), [`--no-unicode-normalization`](/docs/#no-unicode-normalization) ### `v1.64` * Fixed an [issue](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=1.%20Dry%20runs%20are%20not%20completely%20dry)