fs: make sure backends with additional config have a different name #4996

Backends for which additional config is detected (in the config string
or on the command line or as environment variables) will gain a suffix
`{XXXXX}` where `XXXX` is a base64 encoded md5hash of the config
string.

This fixes backend caching with config string remotes.

This much requested feature now works properly:

    rclone copy -vv drive,shared_with_me:file.txt drive:
This commit is contained in:
Nick Craig-Wood 2021-03-10 14:10:03 +00:00
parent 3dbef2b2fd
commit a12b2746b4
3 changed files with 68 additions and 2 deletions

View File

@ -285,6 +285,31 @@ does not work on Windows.)
rclone copy ':http,url="https://example.com":path/to/dir' /tmp/dir
#### Connection strings, config and logging
If you supply extra configuration to a backend by command line flag,
environment variable or connection string then rclone will add a
suffix based on the hash of the config to the name of the remote, eg
rclone -vv lsf --s3-chunk-size 20M s3:
Has the log message
DEBUG : s3: detected overridden config - adding "{Srj1p}" suffix to name
This is so rclone can tell the modified remote apart from the
unmodified remote when caching the backends.
This should only be noticeable in the logs.
This means that on the fly backends such as
rclone -vv lsf :s3,env_auth:
Will get their own names
DEBUG : :s3: detected overridden config - adding "{YTu53}" suffix to name
### Valid remote names
- Remote names may only contain 0-9, A-Z ,a-z ,_ , - and space.

View File

@ -3,6 +3,8 @@ package fs
import (
"context"
"crypto/md5"
"encoding/base64"
"encoding/json"
"fmt"
"io"
@ -1379,6 +1381,34 @@ func NewFs(ctx context.Context, path string) (Fs, error) {
if err != nil {
return nil, err
}
// Now discover which config items have been overridden,
// either by the config string, command line flags or
// environment variables
var overridden = configmap.Simple{}
for i := range fsInfo.Options {
opt := &fsInfo.Options[i]
value, isSet := config.GetOverride(opt.Name)
if isSet {
overridden.Set(opt.Name, value)
}
}
if len(overridden) > 0 {
extraConfig := overridden.String()
//Debugf(nil, "detected overriden config %q", extraConfig)
md5sumBinary := md5.Sum([]byte(extraConfig))
suffix := base64.RawStdEncoding.EncodeToString(md5sumBinary[:])
// 5 characters length is 5*6 = 30 bits of base64
const maxLength = 5
if len(suffix) > maxLength {
suffix = suffix[:maxLength]
}
suffix = "{" + suffix + "}"
Debugf(configName, "detected overridden config - adding %q suffix to name", suffix)
// Add the suffix to the config name
//
// These need to work as filesystem names as the VFS cache will use them
configName += suffix
}
return fsInfo.NewFs(ctx, configName, fsPath, config)
}

View File

@ -318,6 +318,17 @@ func toUpperASCII(s string) string {
}, s)
}
// removeConfigID removes any {xyz} parts of the name put in for
// config disambiguation
func removeConfigID(s string) string {
bra := strings.IndexRune(s, '{')
ket := strings.IndexRune(s, '}')
if bra >= 0 && ket > bra {
s = s[:bra] + s[ket+1:]
}
return s
}
// Run runs the basic integration tests for a remote using the options passed in.
//
// They are structured in a hierarchical way so that dependencies for the tests can be created.
@ -505,7 +516,7 @@ func Run(t *testing.T, opt *Opt) {
// TestFsName tests the Name method
t.Run("FsName", func(t *testing.T) {
skipIfNotOk(t)
got := f.Name()
got := removeConfigID(f.Name())
want := remoteName[:strings.LastIndex(remoteName, ":")+1]
if isLocalRemote {
want = "local:"
@ -516,7 +527,7 @@ func Run(t *testing.T, opt *Opt) {
// TestFsRoot tests the Root method
t.Run("FsRoot", func(t *testing.T) {
skipIfNotOk(t)
name := f.Name() + ":"
name := removeConfigID(f.Name()) + ":"
root := f.Root()
if isLocalRemote {
// only check last path element on local