From 1c4e33d4adc4be6452f7464cad94699e0023d3d4 Mon Sep 17 00:00:00 2001 From: Ivan Andreev Date: Wed, 4 Sep 2019 23:30:48 +0300 Subject: [PATCH] vfs: add flag --vfs-case-insensitive for windows/macOS mounts rclone mount when run on Windows & macOS will now default to `--vfs-case-insensitive`. This means that --- .appveyor.yml | 1 + azure-pipelines.yml | 1 + docs/content/commands/rclone_mount.md | 38 ++++++- vfs/dir.go | 18 +++ vfs/vfs.go | 3 + vfs/vfs_case_test.go | 156 ++++++++++++++++++++++++++ vfs/vfsflags/vfsflags.go | 1 + 7 files changed, 217 insertions(+), 1 deletion(-) create mode 100644 vfs/vfs_case_test.go diff --git a/.appveyor.yml b/.appveyor.yml index bf6d90a00..ed2bd958a 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -40,6 +40,7 @@ build_script: test_script: - make GOTAGS=cmount quicktest +- make GOTAGS=cmount racequicktest artifacts: - path: rclone.exe diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9dff63fdc..137e119fa 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -49,6 +49,7 @@ strategy: GO_VERSION: latest BUILD_FLAGS: '-include "^windows/amd64" -cgo' MAKE_QUICKTEST: true + MAKE_RACEQUICKTEST: true DEPLOY: true windows_386: imageName: windows-2019 diff --git a/docs/content/commands/rclone_mount.md b/docs/content/commands/rclone_mount.md index bdc0f5324..80c7ada11 100644 --- a/docs/content/commands/rclone_mount.md +++ b/docs/content/commands/rclone_mount.md @@ -287,11 +287,46 @@ This mode should support all normal file system operations. If an upload or download fails it will be retried up to --low-level-retries times. - ``` rclone mount remote:path /path/to/mountpoint [flags] ``` +### Case Sensitivity + +Linux file systems are case-sensitive: two files can differ only +by case, and the exact case must be used when opening a file. + +Windows is not like most other operating systems supported by rclone. +File systems in modern Windows are case-insensitive but case-preserving: +although existing files can be opened using any case, the exact case used +to create the file is preserved and available for programs to query. +It is not allowed for two files in the same directory to differ only by case. + +Usually file systems on MacOS are case-insensitive. It is possible to make MacOS +file systems case-sensitive but that is not the default + +The `--vfs-case-insensitive` mount flag controls how rclone handles these +two cases. If its value is `false`, rclone passes file names to the mounted +file system as is. If the flag is `true` (or appears without a value on +command line), rclone may perform a "fixup" as explained below. + +The user may specify a file name to open/delete/rename/etc with a case +different than what is stored on mounted file system. If an argument refers +to an existing file with exactly the same name, then the case of the existing +file on the disk will be used. However, if a file name with exactly the same +name is not found but a name differing only by case exists, rclone will +transparently fixup the name. This fixup happens only when an existing file +is requested. Case sensitivity of file names created anew by rclone is +controlled by an underlying mounted file system. + +Note that case sensitivity of the operating system running rclone (the target) +may differ from case sensitivity of a file system mounted by rclone (the source). +The flag controls whether "fixup" is performed to satisfy the target. + +If the flag is not provided on command line, then its default value depends +on the operating system where rclone runs: `true` on Windows and MacOS, `false` +otherwise. If the flag is provided without a value, then it is `true`. + ### Options ``` @@ -322,6 +357,7 @@ rclone mount remote:path /path/to/mountpoint [flags] --vfs-cache-max-size SizeSuffix Max total size of objects in the cache. (default off) --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) --vfs-cache-poll-interval duration Interval to poll the cache for stale objects. (default 1m0s) + --vfs-case-insensitive [bool] Case insensitive mount true|false (default depends on operating system) --vfs-read-chunk-size SizeSuffix Read the source objects in chunks. (default 128M) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached. 'off' is unlimited. (default off) --volname string Set the volume name (not supported by all OSes). diff --git a/vfs/dir.go b/vfs/dir.go index f36489830..55ab396fe 100644 --- a/vfs/dir.go +++ b/vfs/dir.go @@ -323,6 +323,8 @@ func (d *Dir) readDir() error { // stat a single item in the directory // // returns ENOENT if not found. +// returns a custom error if directory on a case-insensitive file system +// contains files with names that differ only by case. func (d *Dir) stat(leaf string) (Node, error) { d.mu.Lock() defer d.mu.Unlock() @@ -331,6 +333,22 @@ func (d *Dir) stat(leaf string) (Node, error) { return nil, err } item, ok := d.items[leaf] + + if !ok && d.vfs.Opt.CaseInsensitive { + leafLower := strings.ToLower(leaf) + for name, node := range d.items { + if strings.ToLower(name) == leafLower { + if ok { + // duplicate case insensitive match is an error + return nil, errors.Errorf("duplicate filename %q detected with --vfs-case-insensitive set", leaf) + } + // found a case insenstive match + ok = true + item = node + } + } + } + if !ok { return nil, ENOENT } diff --git a/vfs/vfs.go b/vfs/vfs.go index 9fae2fbd0..62324f041 100644 --- a/vfs/vfs.go +++ b/vfs/vfs.go @@ -23,6 +23,7 @@ import ( "fmt" "os" "path" + "runtime" "strings" "sync" "sync/atomic" @@ -51,6 +52,7 @@ var DefaultOpt = Options{ ChunkSize: 128 * fs.MebiByte, ChunkSizeLimit: -1, CacheMaxSize: -1, + CaseInsensitive: runtime.GOOS == "windows" || runtime.GOOS == "darwin", // default to true on Windows and Mac, false otherwise } // Node represents either a directory (*Dir) or a file (*File) @@ -199,6 +201,7 @@ type Options struct { CacheMaxAge time.Duration CacheMaxSize fs.SizeSuffix CachePollInterval time.Duration + CaseInsensitive bool } // New creates a new VFS and root directory. If opt is nil, then diff --git a/vfs/vfs_case_test.go b/vfs/vfs_case_test.go new file mode 100644 index 000000000..96fe576e1 --- /dev/null +++ b/vfs/vfs_case_test.go @@ -0,0 +1,156 @@ +package vfs + +import ( + "context" + "os" + "testing" + + "github.com/rclone/rclone/fstest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCaseSensitivity(t *testing.T) { + r := fstest.NewRun(t) + defer r.Finalise() + + // Create test files + ctx := context.Background() + file1 := r.WriteObject(ctx, "FiLeA", "data1", t1) + file2 := r.WriteObject(ctx, "FiLeB", "data2", t2) + fstest.CheckItems(t, r.Fremote, file1, file2) + + // Create file3 with name differing from file2 name only by case. + // On a case-Sensitive remote this will be a separate file. + // On a case-INsensitive remote this file will either not exist + // or overwrite file2 depending on how file system diverges. + file3 := r.WriteObject(ctx, "FilEb", "data3", t3) + + // Create a case-Sensitive and case-INsensitive VFS + optCS := DefaultOpt + optCS.CaseInsensitive = false + vfsCS := New(r.Fremote, &optCS) + + optCI := DefaultOpt + optCI.CaseInsensitive = true + vfsCI := New(r.Fremote, &optCI) + + // Run basic checks that must pass on VFS of any type. + assertFileDataVFS(t, vfsCI, "FiLeA", "data1") + assertFileDataVFS(t, vfsCS, "FiLeA", "data1") + + // Detect case sensitivity of the underlying remote. + remoteIsOK := true + if !checkFileDataVFS(t, vfsCS, "FiLeA", "data1") { + remoteIsOK = false + } + if !checkFileDataVFS(t, vfsCS, "FiLeB", "data2") { + remoteIsOK = false + } + if !checkFileDataVFS(t, vfsCS, "FilEb", "data3") { + remoteIsOK = false + } + + // The remaining test is only meaningful on a case-Sensitive file system. + if !remoteIsOK { + t.Logf("SKIP: TestCaseSensitivity - remote is not fully case-sensitive") + return + } + + // Continue with test as the underlying remote is fully case-Sensitive. + fstest.CheckItems(t, r.Fremote, file1, file2, file3) + + // See how VFS handles case-INsensitive flag + assertFileDataVFS(t, vfsCI, "FiLeA", "data1") + assertFileDataVFS(t, vfsCI, "fileA", "data1") + assertFileDataVFS(t, vfsCI, "filea", "data1") + assertFileDataVFS(t, vfsCI, "FILEA", "data1") + + assertFileDataVFS(t, vfsCI, "FiLeB", "data2") + assertFileDataVFS(t, vfsCI, "FilEb", "data3") + + fd, err := vfsCI.OpenFile("fileb", os.O_RDONLY, 0777) + assert.Nil(t, fd) + assert.Error(t, err) + assert.NotEqual(t, err, ENOENT) + + fd, err = vfsCI.OpenFile("FILEB", os.O_RDONLY, 0777) + assert.Nil(t, fd) + assert.Error(t, err) + assert.NotEqual(t, err, ENOENT) + + // Run the same set of checks with case-Sensitive VFS, for comparison. + assertFileDataVFS(t, vfsCS, "FiLeA", "data1") + + assertFileAbsentVFS(t, vfsCS, "fileA") + assertFileAbsentVFS(t, vfsCS, "filea") + assertFileAbsentVFS(t, vfsCS, "FILEA") + + assertFileDataVFS(t, vfsCS, "FiLeB", "data2") + assertFileDataVFS(t, vfsCS, "FilEb", "data3") + + assertFileAbsentVFS(t, vfsCS, "fileb") + assertFileAbsentVFS(t, vfsCS, "FILEB") +} + +func checkFileDataVFS(t *testing.T, vfs *VFS, name string, expect string) bool { + fd, err := vfs.OpenFile(name, os.O_RDONLY, 0777) + if fd == nil || err != nil { + return false + } + defer func() { + // File must be closed - otherwise Run.cleanUp() will fail on Windows. + _ = fd.Close() + }() + + fh, ok := fd.(*ReadFileHandle) + if !ok { + return false + } + + size := len(expect) + buf := make([]byte, size) + num, err := fh.Read(buf) + if err != nil || num != size { + return false + } + + return string(buf) == expect +} + +func assertFileDataVFS(t *testing.T, vfs *VFS, name string, expect string) { + fd, errOpen := vfs.OpenFile(name, os.O_RDONLY, 0777) + assert.NotNil(t, fd) + assert.NoError(t, errOpen) + + defer func() { + // File must be closed - otherwise Run.cleanUp() will fail on Windows. + if errOpen == nil && fd != nil { + _ = fd.Close() + } + }() + + fh, ok := fd.(*ReadFileHandle) + require.True(t, ok) + + size := len(expect) + buf := make([]byte, size) + numRead, errRead := fh.Read(buf) + assert.NoError(t, errRead) + assert.Equal(t, numRead, size) + + assert.Equal(t, string(buf), expect) +} + +func assertFileAbsentVFS(t *testing.T, vfs *VFS, name string) { + fd, err := vfs.OpenFile(name, os.O_RDONLY, 0777) + defer func() { + // File must be closed - otherwise Run.cleanUp() will fail on Windows. + if err == nil && fd != nil { + _ = fd.Close() + } + }() + assert.Nil(t, fd) + assert.Error(t, err) + assert.Equal(t, err, ENOENT) +} diff --git a/vfs/vfsflags/vfsflags.go b/vfs/vfsflags/vfsflags.go index f5240f3b9..cf2e910ec 100644 --- a/vfs/vfsflags/vfsflags.go +++ b/vfs/vfsflags/vfsflags.go @@ -32,5 +32,6 @@ func AddFlags(flagSet *pflag.FlagSet) { flags.FVarP(flagSet, &Opt.ChunkSizeLimit, "vfs-read-chunk-size-limit", "", "If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached. 'off' is unlimited.") flags.FVarP(flagSet, DirPerms, "dir-perms", "", "Directory permissions") flags.FVarP(flagSet, FilePerms, "file-perms", "", "File permissions") + flags.BoolVarP(flagSet, &Opt.CaseInsensitive, "vfs-case-insensitive", "", Opt.CaseInsensitive, "If a file name not found, find a case insensitive match.") platformFlags(flagSet) }