sync: --fix-case flag to rename case insensitive dest - fixes #4854

Before this change, a sync to a case insensitive dest (such as macOS / Windows)
would not result in a matching filename if the source and dest had casing
differences but were otherwise equal. For example, syncing `hello.txt` to
`HELLO.txt` would result in the dest filename remaining `HELLO.txt`.
Furthermore, `--local-case-sensitive` did not solve this, as it actually caused
`HELLO.txt` to get deleted!

After this change, `HELLO.txt` is renamed to `hello.txt` to match the source,
only if the `--fix-case` flag is specified. (The old behavior remains the
default.)

docs/content/docs.md

@@ -1107,6 +1107,26 @@ triggering follow-on actions if data was copied, or skipping if not.
 NB: Enabling this option turns a usually non-fatal error into a potentially
 fatal one - please check and adjust your scripts accordingly!
 
+### --fix-case ###
+
+Normally, a sync to a case insensitive dest (such as macOS / Windows) will
+not result in a matching filename if the source and dest filenames have
+casing differences but are otherwise identical. For example, syncing `hello.txt`
+to `HELLO.txt` will normally result in the dest filename remaining `HELLO.txt`.
+If `--fix-case` is set, then `HELLO.txt` will be renamed to `hello.txt`
+to match the source.
+
+NB:
+- directory names with incorrect casing will also be fixed
+- `--fix-case` will be ignored if `--immutable` is set
+- using `--local-case-sensitive` instead is not advisable;
+it will cause `HELLO.txt` to get deleted!
+- the old dest filename must not be excluded by filters.
+Be especially careful with [`--files-from`](/filtering/#files-from-read-list-of-source-file-names),
+which does not respect [`--ignore-case`](/filtering/#ignore-case-make-searches-case-insensitive)!
+- on remotes that do not support server-side move, `--fix-case` will require
+downloading the file and re-uploading it. To avoid this, do not use `--fix-case`.
+
 ### --fs-cache-expire-duration=TIME
 
 When using rclone via the API rclone caches created remotes for 5

fs/config.go

@@ -81,6 +81,7 @@ type ConfigInfo struct {
 	IgnoreSize                 bool
 	IgnoreChecksum             bool
 	IgnoreCaseSync             bool
+	FixCase                    bool
 	NoTraverse                 bool
 	CheckFirst                 bool
 	NoCheckDest                bool

fs/config/configflags/configflags.go

@@ -83,6 +83,7 @@ func AddFlags(ci *fs.ConfigInfo, flagSet *pflag.FlagSet) {
 	flags.BoolVarP(flagSet, &ci.IgnoreSize, "ignore-size", "", false, "Ignore size when skipping use modtime or checksum", "Copy")
 	flags.BoolVarP(flagSet, &ci.IgnoreChecksum, "ignore-checksum", "", ci.IgnoreChecksum, "Skip post copy check of checksums", "Copy")
 	flags.BoolVarP(flagSet, &ci.IgnoreCaseSync, "ignore-case-sync", "", ci.IgnoreCaseSync, "Ignore case when synchronizing", "Copy")
+	flags.BoolVarP(flagSet, &ci.FixCase, "fix-case", "", ci.FixCase, "Force rename of case insensitive dest to match source", "Sync")
 	flags.BoolVarP(flagSet, &ci.NoTraverse, "no-traverse", "", ci.NoTraverse, "Don't traverse destination file system on copy", "Copy")
 	flags.BoolVarP(flagSet, &ci.CheckFirst, "check-first", "", ci.CheckFirst, "Do all the checks before starting transfers", "Copy")
 	flags.BoolVarP(flagSet, &ci.NoCheckDest, "no-check-dest", "", ci.NoCheckDest, "Don't check the destination, copy regardless", "Copy")

fs/sync/sync.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"path"
+	"path/filepath"
 	"sort"
 	"strings"
 	"sync"
@@ -18,6 +19,7 @@ import (
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fs/march"
 	"github.com/rclone/rclone/fs/operations"
+	"github.com/rclone/rclone/lib/random"
 )
 
 // ErrorMaxDurationReached defines error when transfer duration is reached
@@ -363,6 +365,16 @@ func (s *syncCopyMove) pairChecker(in *pipe, out *pipe, fraction int, wg *sync.W
 					needTransfer = false
 				}
 			}
+			// Fix case for case insensitive filesystems
+			if s.ci.FixCase && !s.ci.Immutable && src.Remote() != pair.Dst.Remote() {
+				if newDst, err := operations.Move(s.ctx, s.fdst, nil, src.Remote(), pair.Dst); err != nil {
+					fs.Errorf(pair.Dst, "Error while attempting to rename to %s: %v", src.Remote(), err)
+					s.processError(err)
+				} else {
+					fs.Infof(pair.Dst, "Fixed case by renaming to: %s", src.Remote())
+					pair.Dst = newDst
+				}
+			}
 			if needTransfer {
 				// If files are treated as immutable, fail if destination exists and does not match
 				if s.ci.Immutable && pair.Dst != nil {
@@ -1136,6 +1148,30 @@ func (s *syncCopyMove) Match(ctx context.Context, dst, src fs.DirEntry) (recurse
 				s.srcEmptyDirs[src.Remote()] = src
 				s.srcEmptyDirsMu.Unlock()
 			}
+			if s.ci.FixCase && !s.ci.Immutable && src.Remote() != dst.Remote() {
+				// Fix case for case insensitive filesystems
+				// Fix each dir before recursing into subdirs and files
+				oldDirFs, err := fs.NewFs(s.ctx, filepath.Join(s.fdst.Root(), dst.Remote()))
+				s.processError(err)
+				newDirPath := filepath.Join(s.fdst.Root(), filepath.Dir(dst.Remote()), filepath.Base(src.Remote()))
+				newDirFs, err := fs.NewFs(s.ctx, newDirPath)
+				s.processError(err)
+				// Create random name to temporarily move dir to
+				tmpDirName := newDirPath + "-rclone-move-" + random.String(8)
+				tmpDirFs, err := fs.NewFs(s.ctx, tmpDirName)
+				s.processError(err)
+				if err = MoveDir(s.ctx, tmpDirFs, oldDirFs, s.deleteEmptySrcDirs, s.copyEmptySrcDirs); err != nil {
+					fs.Errorf(dst, "Error while attempting to move dir to temporary location %s: %v", tmpDirName, err)
+					s.processError(err)
+				} else {
+					if err = MoveDir(s.ctx, newDirFs, tmpDirFs, s.deleteEmptySrcDirs, s.copyEmptySrcDirs); err != nil {
+						fs.Errorf(dst, "Error while attempting to rename to %s: %v", src.Remote(), err)
+						s.processError(err)
+					} else {
+						fs.Infof(dst, "Fixed case by renaming to: %s", src.Remote())
+					}
+				}
+			}
 
 			return true
 		}

fs/sync/sync_test.go

@@ -2197,6 +2197,39 @@ func TestSyncIgnoreCase(t *testing.T) {
 	r.CheckRemoteItems(t, file2)
 }
 
+// Test --fix-case
+func TestFixCase(t *testing.T) {
+	ctx := context.Background()
+	ctx, ci := fs.AddConfig(ctx)
+	r := fstest.NewRun(t)
+
+	// Only test if remote is case insensitive
+	if !r.Fremote.Features().CaseInsensitive {
+		t.Skip("Skipping test as local or remote are case-sensitive")
+	}
+
+	ci.FixCase = true
+
+	// Create files with different filename casing
+	file1a := r.WriteFile("existing", "potato", t1)
+	file1b := r.WriteFile("existingbutdifferent", "donut", t1)
+	file1c := r.WriteFile("subdira/subdirb/subdirc/hello", "donut", t1)
+	file1d := r.WriteFile("subdira/subdirb/subdirc/subdird/filewithoutcasedifferences", "donut", t1)
+	r.CheckLocalItems(t, file1a, file1b, file1c, file1d)
+	file2a := r.WriteObject(ctx, "EXISTING", "potato", t1)
+	file2b := r.WriteObject(ctx, "EXISTINGBUTDIFFERENT", "lemonade", t1)
+	file2c := r.WriteObject(ctx, "SUBDIRA/subdirb/SUBDIRC/HELLO", "lemonade", t1)
+	file2d := r.WriteObject(ctx, "SUBDIRA/subdirb/SUBDIRC/subdird/filewithoutcasedifferences", "lemonade", t1)
+	r.CheckRemoteItems(t, file2a, file2b, file2c, file2d)
+
+	// Should force rename of dest file that is differently-cased
+	accounting.GlobalStats().ResetCounters()
+	err := Sync(ctx, r.Fremote, r.Flocal, false)
+	require.NoError(t, err)
+	r.CheckLocalItems(t, file1a, file1b, file1c, file1d)
+	r.CheckRemoteItems(t, file1a, file1b, file1c, file1d)
+}
+
 // Test that aborting on --max-transfer works
 func TestMaxTransfer(t *testing.T) {
 	ctx := context.Background()