vfs: add flag --vfs-case-insensitive for windows/macOS mounts

rclone mount when run on Windows & macOS will now default to `--vfs-case-insensitive`.
This means that
This commit is contained in:
Ivan Andreev 2019-09-04 23:30:48 +03:00 committed by Nick Craig-Wood
parent 530ba66d35
commit 1c4e33d4ad
7 changed files with 217 additions and 1 deletions

View File

@ -40,6 +40,7 @@ build_script:
test_script: test_script:
- make GOTAGS=cmount quicktest - make GOTAGS=cmount quicktest
- make GOTAGS=cmount racequicktest
artifacts: artifacts:
- path: rclone.exe - path: rclone.exe

View File

@ -49,6 +49,7 @@ strategy:
GO_VERSION: latest GO_VERSION: latest
BUILD_FLAGS: '-include "^windows/amd64" -cgo' BUILD_FLAGS: '-include "^windows/amd64" -cgo'
MAKE_QUICKTEST: true MAKE_QUICKTEST: true
MAKE_RACEQUICKTEST: true
DEPLOY: true DEPLOY: true
windows_386: windows_386:
imageName: windows-2019 imageName: windows-2019

View File

@ -287,11 +287,46 @@ This mode should support all normal file system operations.
If an upload or download fails it will be retried up to If an upload or download fails it will be retried up to
--low-level-retries times. --low-level-retries times.
``` ```
rclone mount remote:path /path/to/mountpoint [flags] rclone mount remote:path /path/to/mountpoint [flags]
``` ```
### Case Sensitivity
Linux file systems are case-sensitive: two files can differ only
by case, and the exact case must be used when opening a file.
Windows is not like most other operating systems supported by rclone.
File systems in modern Windows are case-insensitive but case-preserving:
although existing files can be opened using any case, the exact case used
to create the file is preserved and available for programs to query.
It is not allowed for two files in the same directory to differ only by case.
Usually file systems on MacOS are case-insensitive. It is possible to make MacOS
file systems case-sensitive but that is not the default
The `--vfs-case-insensitive` mount flag controls how rclone handles these
two cases. If its value is `false`, rclone passes file names to the mounted
file system as is. If the flag is `true` (or appears without a value on
command line), rclone may perform a "fixup" as explained below.
The user may specify a file name to open/delete/rename/etc with a case
different than what is stored on mounted file system. If an argument refers
to an existing file with exactly the same name, then the case of the existing
file on the disk will be used. However, if a file name with exactly the same
name is not found but a name differing only by case exists, rclone will
transparently fixup the name. This fixup happens only when an existing file
is requested. Case sensitivity of file names created anew by rclone is
controlled by an underlying mounted file system.
Note that case sensitivity of the operating system running rclone (the target)
may differ from case sensitivity of a file system mounted by rclone (the source).
The flag controls whether "fixup" is performed to satisfy the target.
If the flag is not provided on command line, then its default value depends
on the operating system where rclone runs: `true` on Windows and MacOS, `false`
otherwise. If the flag is provided without a value, then it is `true`.
### Options ### Options
``` ```
@ -322,6 +357,7 @@ rclone mount remote:path /path/to/mountpoint [flags]
--vfs-cache-max-size SizeSuffix Max total size of objects in the cache. (default off) --vfs-cache-max-size SizeSuffix Max total size of objects in the cache. (default off)
--vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off)
--vfs-cache-poll-interval duration Interval to poll the cache for stale objects. (default 1m0s) --vfs-cache-poll-interval duration Interval to poll the cache for stale objects. (default 1m0s)
--vfs-case-insensitive [bool] Case insensitive mount true|false (default depends on operating system)
--vfs-read-chunk-size SizeSuffix Read the source objects in chunks. (default 128M) --vfs-read-chunk-size SizeSuffix Read the source objects in chunks. (default 128M)
--vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached. 'off' is unlimited. (default off) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached. 'off' is unlimited. (default off)
--volname string Set the volume name (not supported by all OSes). --volname string Set the volume name (not supported by all OSes).

View File

@ -323,6 +323,8 @@ func (d *Dir) readDir() error {
// stat a single item in the directory // stat a single item in the directory
// //
// returns ENOENT if not found. // returns ENOENT if not found.
// returns a custom error if directory on a case-insensitive file system
// contains files with names that differ only by case.
func (d *Dir) stat(leaf string) (Node, error) { func (d *Dir) stat(leaf string) (Node, error) {
d.mu.Lock() d.mu.Lock()
defer d.mu.Unlock() defer d.mu.Unlock()
@ -331,6 +333,22 @@ func (d *Dir) stat(leaf string) (Node, error) {
return nil, err return nil, err
} }
item, ok := d.items[leaf] item, ok := d.items[leaf]
if !ok && d.vfs.Opt.CaseInsensitive {
leafLower := strings.ToLower(leaf)
for name, node := range d.items {
if strings.ToLower(name) == leafLower {
if ok {
// duplicate case insensitive match is an error
return nil, errors.Errorf("duplicate filename %q detected with --vfs-case-insensitive set", leaf)
}
// found a case insenstive match
ok = true
item = node
}
}
}
if !ok { if !ok {
return nil, ENOENT return nil, ENOENT
} }

View File

@ -23,6 +23,7 @@ import (
"fmt" "fmt"
"os" "os"
"path" "path"
"runtime"
"strings" "strings"
"sync" "sync"
"sync/atomic" "sync/atomic"
@ -51,6 +52,7 @@ var DefaultOpt = Options{
ChunkSize: 128 * fs.MebiByte, ChunkSize: 128 * fs.MebiByte,
ChunkSizeLimit: -1, ChunkSizeLimit: -1,
CacheMaxSize: -1, CacheMaxSize: -1,
CaseInsensitive: runtime.GOOS == "windows" || runtime.GOOS == "darwin", // default to true on Windows and Mac, false otherwise
} }
// Node represents either a directory (*Dir) or a file (*File) // Node represents either a directory (*Dir) or a file (*File)
@ -199,6 +201,7 @@ type Options struct {
CacheMaxAge time.Duration CacheMaxAge time.Duration
CacheMaxSize fs.SizeSuffix CacheMaxSize fs.SizeSuffix
CachePollInterval time.Duration CachePollInterval time.Duration
CaseInsensitive bool
} }
// New creates a new VFS and root directory. If opt is nil, then // New creates a new VFS and root directory. If opt is nil, then

156
vfs/vfs_case_test.go Normal file
View File

@ -0,0 +1,156 @@
package vfs
import (
"context"
"os"
"testing"
"github.com/rclone/rclone/fstest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestCaseSensitivity(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
// Create test files
ctx := context.Background()
file1 := r.WriteObject(ctx, "FiLeA", "data1", t1)
file2 := r.WriteObject(ctx, "FiLeB", "data2", t2)
fstest.CheckItems(t, r.Fremote, file1, file2)
// Create file3 with name differing from file2 name only by case.
// On a case-Sensitive remote this will be a separate file.
// On a case-INsensitive remote this file will either not exist
// or overwrite file2 depending on how file system diverges.
file3 := r.WriteObject(ctx, "FilEb", "data3", t3)
// Create a case-Sensitive and case-INsensitive VFS
optCS := DefaultOpt
optCS.CaseInsensitive = false
vfsCS := New(r.Fremote, &optCS)
optCI := DefaultOpt
optCI.CaseInsensitive = true
vfsCI := New(r.Fremote, &optCI)
// Run basic checks that must pass on VFS of any type.
assertFileDataVFS(t, vfsCI, "FiLeA", "data1")
assertFileDataVFS(t, vfsCS, "FiLeA", "data1")
// Detect case sensitivity of the underlying remote.
remoteIsOK := true
if !checkFileDataVFS(t, vfsCS, "FiLeA", "data1") {
remoteIsOK = false
}
if !checkFileDataVFS(t, vfsCS, "FiLeB", "data2") {
remoteIsOK = false
}
if !checkFileDataVFS(t, vfsCS, "FilEb", "data3") {
remoteIsOK = false
}
// The remaining test is only meaningful on a case-Sensitive file system.
if !remoteIsOK {
t.Logf("SKIP: TestCaseSensitivity - remote is not fully case-sensitive")
return
}
// Continue with test as the underlying remote is fully case-Sensitive.
fstest.CheckItems(t, r.Fremote, file1, file2, file3)
// See how VFS handles case-INsensitive flag
assertFileDataVFS(t, vfsCI, "FiLeA", "data1")
assertFileDataVFS(t, vfsCI, "fileA", "data1")
assertFileDataVFS(t, vfsCI, "filea", "data1")
assertFileDataVFS(t, vfsCI, "FILEA", "data1")
assertFileDataVFS(t, vfsCI, "FiLeB", "data2")
assertFileDataVFS(t, vfsCI, "FilEb", "data3")
fd, err := vfsCI.OpenFile("fileb", os.O_RDONLY, 0777)
assert.Nil(t, fd)
assert.Error(t, err)
assert.NotEqual(t, err, ENOENT)
fd, err = vfsCI.OpenFile("FILEB", os.O_RDONLY, 0777)
assert.Nil(t, fd)
assert.Error(t, err)
assert.NotEqual(t, err, ENOENT)
// Run the same set of checks with case-Sensitive VFS, for comparison.
assertFileDataVFS(t, vfsCS, "FiLeA", "data1")
assertFileAbsentVFS(t, vfsCS, "fileA")
assertFileAbsentVFS(t, vfsCS, "filea")
assertFileAbsentVFS(t, vfsCS, "FILEA")
assertFileDataVFS(t, vfsCS, "FiLeB", "data2")
assertFileDataVFS(t, vfsCS, "FilEb", "data3")
assertFileAbsentVFS(t, vfsCS, "fileb")
assertFileAbsentVFS(t, vfsCS, "FILEB")
}
func checkFileDataVFS(t *testing.T, vfs *VFS, name string, expect string) bool {
fd, err := vfs.OpenFile(name, os.O_RDONLY, 0777)
if fd == nil || err != nil {
return false
}
defer func() {
// File must be closed - otherwise Run.cleanUp() will fail on Windows.
_ = fd.Close()
}()
fh, ok := fd.(*ReadFileHandle)
if !ok {
return false
}
size := len(expect)
buf := make([]byte, size)
num, err := fh.Read(buf)
if err != nil || num != size {
return false
}
return string(buf) == expect
}
func assertFileDataVFS(t *testing.T, vfs *VFS, name string, expect string) {
fd, errOpen := vfs.OpenFile(name, os.O_RDONLY, 0777)
assert.NotNil(t, fd)
assert.NoError(t, errOpen)
defer func() {
// File must be closed - otherwise Run.cleanUp() will fail on Windows.
if errOpen == nil && fd != nil {
_ = fd.Close()
}
}()
fh, ok := fd.(*ReadFileHandle)
require.True(t, ok)
size := len(expect)
buf := make([]byte, size)
numRead, errRead := fh.Read(buf)
assert.NoError(t, errRead)
assert.Equal(t, numRead, size)
assert.Equal(t, string(buf), expect)
}
func assertFileAbsentVFS(t *testing.T, vfs *VFS, name string) {
fd, err := vfs.OpenFile(name, os.O_RDONLY, 0777)
defer func() {
// File must be closed - otherwise Run.cleanUp() will fail on Windows.
if err == nil && fd != nil {
_ = fd.Close()
}
}()
assert.Nil(t, fd)
assert.Error(t, err)
assert.Equal(t, err, ENOENT)
}

View File

@ -32,5 +32,6 @@ func AddFlags(flagSet *pflag.FlagSet) {
flags.FVarP(flagSet, &Opt.ChunkSizeLimit, "vfs-read-chunk-size-limit", "", "If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached. 'off' is unlimited.") flags.FVarP(flagSet, &Opt.ChunkSizeLimit, "vfs-read-chunk-size-limit", "", "If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached. 'off' is unlimited.")
flags.FVarP(flagSet, DirPerms, "dir-perms", "", "Directory permissions") flags.FVarP(flagSet, DirPerms, "dir-perms", "", "Directory permissions")
flags.FVarP(flagSet, FilePerms, "file-perms", "", "File permissions") flags.FVarP(flagSet, FilePerms, "file-perms", "", "File permissions")
flags.BoolVarP(flagSet, &Opt.CaseInsensitive, "vfs-case-insensitive", "", Opt.CaseInsensitive, "If a file name not found, find a case insensitive match.")
platformFlags(flagSet) platformFlags(flagSet)
} }