diff --git a/docs/content/docs.md b/docs/content/docs.md index 24d60f22b..101b14eed 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -285,6 +285,31 @@ does not work on Windows.) rclone copy ':http,url="https://example.com":path/to/dir' /tmp/dir +#### Connection strings, config and logging + +If you supply extra configuration to a backend by command line flag, +environment variable or connection string then rclone will add a +suffix based on the hash of the config to the name of the remote, eg + + rclone -vv lsf --s3-chunk-size 20M s3: + +Has the log message + + DEBUG : s3: detected overridden config - adding "{Srj1p}" suffix to name + +This is so rclone can tell the modified remote apart from the +unmodified remote when caching the backends. + +This should only be noticeable in the logs. + +This means that on the fly backends such as + + rclone -vv lsf :s3,env_auth: + +Will get their own names + + DEBUG : :s3: detected overridden config - adding "{YTu53}" suffix to name + ### Valid remote names - Remote names may only contain 0-9, A-Z ,a-z ,_ , - and space. diff --git a/fs/fs.go b/fs/fs.go index a833a6a07..fa0d87722 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -3,6 +3,8 @@ package fs import ( "context" + "crypto/md5" + "encoding/base64" "encoding/json" "fmt" "io" @@ -1379,6 +1381,34 @@ func NewFs(ctx context.Context, path string) (Fs, error) { if err != nil { return nil, err } + // Now discover which config items have been overridden, + // either by the config string, command line flags or + // environment variables + var overridden = configmap.Simple{} + for i := range fsInfo.Options { + opt := &fsInfo.Options[i] + value, isSet := config.GetOverride(opt.Name) + if isSet { + overridden.Set(opt.Name, value) + } + } + if len(overridden) > 0 { + extraConfig := overridden.String() + //Debugf(nil, "detected overriden config %q", extraConfig) + md5sumBinary := md5.Sum([]byte(extraConfig)) + suffix := base64.RawStdEncoding.EncodeToString(md5sumBinary[:]) + // 5 characters length is 5*6 = 30 bits of base64 + const maxLength = 5 + if len(suffix) > maxLength { + suffix = suffix[:maxLength] + } + suffix = "{" + suffix + "}" + Debugf(configName, "detected overridden config - adding %q suffix to name", suffix) + // Add the suffix to the config name + // + // These need to work as filesystem names as the VFS cache will use them + configName += suffix + } return fsInfo.NewFs(ctx, configName, fsPath, config) } diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go index 6ddf1ca54..332226775 100644 --- a/fstest/fstests/fstests.go +++ b/fstest/fstests/fstests.go @@ -318,6 +318,17 @@ func toUpperASCII(s string) string { }, s) } +// removeConfigID removes any {xyz} parts of the name put in for +// config disambiguation +func removeConfigID(s string) string { + bra := strings.IndexRune(s, '{') + ket := strings.IndexRune(s, '}') + if bra >= 0 && ket > bra { + s = s[:bra] + s[ket+1:] + } + return s +} + // Run runs the basic integration tests for a remote using the options passed in. // // They are structured in a hierarchical way so that dependencies for the tests can be created. @@ -505,7 +516,7 @@ func Run(t *testing.T, opt *Opt) { // TestFsName tests the Name method t.Run("FsName", func(t *testing.T) { skipIfNotOk(t) - got := f.Name() + got := removeConfigID(f.Name()) want := remoteName[:strings.LastIndex(remoteName, ":")+1] if isLocalRemote { want = "local:" @@ -516,7 +527,7 @@ func Run(t *testing.T, opt *Opt) { // TestFsRoot tests the Root method t.Run("FsRoot", func(t *testing.T) { skipIfNotOk(t) - name := f.Name() + ":" + name := removeConfigID(f.Name()) + ":" root := f.Root() if isLocalRemote { // only check last path element on local