From 58d82a5c7302e09831060422a8a53ddaa5ab65c1 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 30 Mar 2021 16:12:46 +0100 Subject: [PATCH] rc: allow fs= params to be a JSON blob --- docs/content/rc.md | 49 +++++++++++++++++++++++ fs/rc/cache.go | 47 +++++++++++++++++++++- fs/rc/cache_test.go | 95 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 189 insertions(+), 2 deletions(-) diff --git a/docs/content/rc.md b/docs/content/rc.md index ffb37369b..8ef29ad29 100644 --- a/docs/content/rc.md +++ b/docs/content/rc.md @@ -378,6 +378,55 @@ call and taken by the [options/set](#options-set) calls as well as the - `BandwidthSpec` - this will be set and returned as a string, eg "1M". +## Specifying remotes to work on + +Remotes are specified with the `fs=`, `srcFs=`, `dstFs=` +parameters depending on the command being used. + +The parameters can be a string as per the rest of rclone, eg +`s3:bucket/path` or `:sftp:/my/dir`. They can also be specified as +JSON blobs. + +If specifyng a JSON blob it should be a object mapping strings to +strings. These values will be used to configure the remote. There are +3 special values which may be set: + +- `type` - set to `type` to specify a remote called `:type:` +- `_name` - set to `name` to specify a remote called `name:` +- `_root` - sets the root of the remote - may be empty + +One of `_name` or `type` should normally be set. If the `local` +backend is desired then `type` should be set to `local`. If `_root` +isn't specified then it defaults to the root of the remote. + +For example this JSON is equivalent to `remote:/tmp` + +``` +{ + "_name": "remote", + "_path": "/tmp" +} +``` + +And this is equivalent to `:sftp,host='example.com':/tmp` + +``` +{ + "type": "sftp", + "host": "example.com", + "_path": "/tmp" +} +``` + +And this is equivalent to `/tmp/dir` + +``` +{ + type = "local", + _ path = "/tmp/dir" +} +``` + ## Supported commands {{< rem autogenerated start "- run make rcdocs - don't edit here" >}} ### backend/command: Runs a backend command. {#backend-command} diff --git a/fs/rc/cache.go b/fs/rc/cache.go index 989e9ffe7..062a9ce3e 100644 --- a/fs/rc/cache.go +++ b/fs/rc/cache.go @@ -4,21 +4,64 @@ package rc import ( "context" + "errors" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/cache" + "github.com/rclone/rclone/fs/config/configmap" ) // GetFsNamed gets an fs.Fs named fsName either from the cache or creates it afresh func GetFsNamed(ctx context.Context, in Params, fsName string) (f fs.Fs, err error) { fsString, err := in.GetString(fsName) if err != nil { - return nil, err + if !IsErrParamInvalid(err) { + return nil, err + } + fsString, err = getConfigMap(in, fsName) + if err != nil { + return nil, err + } } - return cache.Get(ctx, fsString) } +// getConfigMap gets the config as a map from in and converts it to a +// config string +// +// It uses the special parameters _name to name the remote and _root +// to make the root of the remote. +func getConfigMap(in Params, fsName string) (fsString string, err error) { + var m configmap.Simple + err = in.GetStruct(fsName, &m) + if err != nil { + return fsString, err + } + pop := func(key string) string { + value := m[key] + delete(m, key) + return value + } + Type := pop("type") + name := pop("_name") + root := pop("_root") + if name != "" { + fsString = name + } else if Type != "" { + fsString = ":" + Type + } else { + return fsString, errors.New(`couldn't find "type" or "_name" in JSON config definition`) + } + config := m.String() + if config != "" { + fsString += "," + fsString += config + } + fsString += ":" + fsString += root + return fsString, nil +} + // GetFs gets an fs.Fs named "fs" either from the cache or creates it afresh func GetFs(ctx context.Context, in Params) (f fs.Fs, err error) { return GetFsNamed(ctx, in, "fs") diff --git a/fs/rc/cache_test.go b/fs/rc/cache_test.go index e81d493aa..bbda7e0b6 100644 --- a/fs/rc/cache_test.go +++ b/fs/rc/cache_test.go @@ -2,6 +2,7 @@ package rc import ( "context" + "fmt" "testing" "github.com/rclone/rclone/fs/cache" @@ -13,6 +14,8 @@ import ( func mockNewFs(t *testing.T) func() { f := mockfs.NewFs(context.Background(), "mock", "mock") cache.Put("/", f) + cache.Put("mock:/", f) + cache.Put(":mock:/", f) return func() { cache.Clear() } @@ -36,6 +39,98 @@ func TestGetFsNamed(t *testing.T) { assert.Nil(t, f) } +func TestGetFsNamedStruct(t *testing.T) { + defer mockNewFs(t)() + + in := Params{ + "potato": Params{ + "type": "mock", + "_root": "/", + }, + } + f, err := GetFsNamed(context.Background(), in, "potato") + require.NoError(t, err) + assert.NotNil(t, f) + + in = Params{ + "potato": Params{ + "_name": "mock", + "_root": "/", + }, + } + f, err = GetFsNamed(context.Background(), in, "potato") + require.NoError(t, err) + assert.NotNil(t, f) +} + +func TestGetConfigMap(t *testing.T) { + for _, test := range []struct { + in Params + fsName string + wantFsString string + wantErr string + }{ + { + in: Params{ + "Fs": Params{}, + }, + fsName: "Fs", + wantErr: `couldn't find "type" or "_name" in JSON config definition`, + }, + { + in: Params{ + "Fs": Params{ + "notastring": true, + }, + }, + fsName: "Fs", + wantErr: `cannot unmarshal bool`, + }, + { + in: Params{ + "Fs": Params{ + "_name": "potato", + }, + }, + fsName: "Fs", + wantFsString: "potato:", + }, + { + in: Params{ + "Fs": Params{ + "type": "potato", + }, + }, + fsName: "Fs", + wantFsString: ":potato:", + }, + { + in: Params{ + "Fs": Params{ + "type": "sftp", + "_name": "potato", + "parameter": "42", + "parameter2": "true", + "_root": "/path/to/somewhere", + }, + }, + fsName: "Fs", + wantFsString: "potato,parameter='42',parameter2='true':/path/to/somewhere", + }, + } { + gotFsString, gotErr := getConfigMap(test.in, test.fsName) + what := fmt.Sprintf("%+v", test.in) + assert.Equal(t, test.wantFsString, gotFsString, what) + if test.wantErr == "" { + assert.NoError(t, gotErr) + } else { + require.Error(t, gotErr) + assert.Contains(t, gotErr.Error(), test.wantErr) + + } + } +} + func TestGetFs(t *testing.T) { defer mockNewFs(t)()