rc: allow fs= params to be a JSON blob

This commit is contained in:
Nick Craig-Wood 2021-03-30 16:12:46 +01:00
parent c0c74003f2
commit 58d82a5c73
3 changed files with 189 additions and 2 deletions

View File

@ -378,6 +378,55 @@ call and taken by the [options/set](#options-set) calls as well as the
- `BandwidthSpec` - this will be set and returned as a string, eg - `BandwidthSpec` - this will be set and returned as a string, eg
"1M". "1M".
## Specifying remotes to work on
Remotes are specified with the `fs=`, `srcFs=`, `dstFs=`
parameters depending on the command being used.
The parameters can be a string as per the rest of rclone, eg
`s3:bucket/path` or `:sftp:/my/dir`. They can also be specified as
JSON blobs.
If specifyng a JSON blob it should be a object mapping strings to
strings. These values will be used to configure the remote. There are
3 special values which may be set:
- `type` - set to `type` to specify a remote called `:type:`
- `_name` - set to `name` to specify a remote called `name:`
- `_root` - sets the root of the remote - may be empty
One of `_name` or `type` should normally be set. If the `local`
backend is desired then `type` should be set to `local`. If `_root`
isn't specified then it defaults to the root of the remote.
For example this JSON is equivalent to `remote:/tmp`
```
{
"_name": "remote",
"_path": "/tmp"
}
```
And this is equivalent to `:sftp,host='example.com':/tmp`
```
{
"type": "sftp",
"host": "example.com",
"_path": "/tmp"
}
```
And this is equivalent to `/tmp/dir`
```
{
type = "local",
_ path = "/tmp/dir"
}
```
## Supported commands ## Supported commands
{{< rem autogenerated start "- run make rcdocs - don't edit here" >}} {{< rem autogenerated start "- run make rcdocs - don't edit here" >}}
### backend/command: Runs a backend command. {#backend-command} ### backend/command: Runs a backend command. {#backend-command}

View File

@ -4,21 +4,64 @@ package rc
import ( import (
"context" "context"
"errors"
"github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/cache" "github.com/rclone/rclone/fs/cache"
"github.com/rclone/rclone/fs/config/configmap"
) )
// GetFsNamed gets an fs.Fs named fsName either from the cache or creates it afresh // GetFsNamed gets an fs.Fs named fsName either from the cache or creates it afresh
func GetFsNamed(ctx context.Context, in Params, fsName string) (f fs.Fs, err error) { func GetFsNamed(ctx context.Context, in Params, fsName string) (f fs.Fs, err error) {
fsString, err := in.GetString(fsName) fsString, err := in.GetString(fsName)
if err != nil { if err != nil {
return nil, err if !IsErrParamInvalid(err) {
return nil, err
}
fsString, err = getConfigMap(in, fsName)
if err != nil {
return nil, err
}
} }
return cache.Get(ctx, fsString) return cache.Get(ctx, fsString)
} }
// getConfigMap gets the config as a map from in and converts it to a
// config string
//
// It uses the special parameters _name to name the remote and _root
// to make the root of the remote.
func getConfigMap(in Params, fsName string) (fsString string, err error) {
var m configmap.Simple
err = in.GetStruct(fsName, &m)
if err != nil {
return fsString, err
}
pop := func(key string) string {
value := m[key]
delete(m, key)
return value
}
Type := pop("type")
name := pop("_name")
root := pop("_root")
if name != "" {
fsString = name
} else if Type != "" {
fsString = ":" + Type
} else {
return fsString, errors.New(`couldn't find "type" or "_name" in JSON config definition`)
}
config := m.String()
if config != "" {
fsString += ","
fsString += config
}
fsString += ":"
fsString += root
return fsString, nil
}
// GetFs gets an fs.Fs named "fs" either from the cache or creates it afresh // GetFs gets an fs.Fs named "fs" either from the cache or creates it afresh
func GetFs(ctx context.Context, in Params) (f fs.Fs, err error) { func GetFs(ctx context.Context, in Params) (f fs.Fs, err error) {
return GetFsNamed(ctx, in, "fs") return GetFsNamed(ctx, in, "fs")

View File

@ -2,6 +2,7 @@ package rc
import ( import (
"context" "context"
"fmt"
"testing" "testing"
"github.com/rclone/rclone/fs/cache" "github.com/rclone/rclone/fs/cache"
@ -13,6 +14,8 @@ import (
func mockNewFs(t *testing.T) func() { func mockNewFs(t *testing.T) func() {
f := mockfs.NewFs(context.Background(), "mock", "mock") f := mockfs.NewFs(context.Background(), "mock", "mock")
cache.Put("/", f) cache.Put("/", f)
cache.Put("mock:/", f)
cache.Put(":mock:/", f)
return func() { return func() {
cache.Clear() cache.Clear()
} }
@ -36,6 +39,98 @@ func TestGetFsNamed(t *testing.T) {
assert.Nil(t, f) assert.Nil(t, f)
} }
func TestGetFsNamedStruct(t *testing.T) {
defer mockNewFs(t)()
in := Params{
"potato": Params{
"type": "mock",
"_root": "/",
},
}
f, err := GetFsNamed(context.Background(), in, "potato")
require.NoError(t, err)
assert.NotNil(t, f)
in = Params{
"potato": Params{
"_name": "mock",
"_root": "/",
},
}
f, err = GetFsNamed(context.Background(), in, "potato")
require.NoError(t, err)
assert.NotNil(t, f)
}
func TestGetConfigMap(t *testing.T) {
for _, test := range []struct {
in Params
fsName string
wantFsString string
wantErr string
}{
{
in: Params{
"Fs": Params{},
},
fsName: "Fs",
wantErr: `couldn't find "type" or "_name" in JSON config definition`,
},
{
in: Params{
"Fs": Params{
"notastring": true,
},
},
fsName: "Fs",
wantErr: `cannot unmarshal bool`,
},
{
in: Params{
"Fs": Params{
"_name": "potato",
},
},
fsName: "Fs",
wantFsString: "potato:",
},
{
in: Params{
"Fs": Params{
"type": "potato",
},
},
fsName: "Fs",
wantFsString: ":potato:",
},
{
in: Params{
"Fs": Params{
"type": "sftp",
"_name": "potato",
"parameter": "42",
"parameter2": "true",
"_root": "/path/to/somewhere",
},
},
fsName: "Fs",
wantFsString: "potato,parameter='42',parameter2='true':/path/to/somewhere",
},
} {
gotFsString, gotErr := getConfigMap(test.in, test.fsName)
what := fmt.Sprintf("%+v", test.in)
assert.Equal(t, test.wantFsString, gotFsString, what)
if test.wantErr == "" {
assert.NoError(t, gotErr)
} else {
require.Error(t, gotErr)
assert.Contains(t, gotErr.Error(), test.wantErr)
}
}
}
func TestGetFs(t *testing.T) { func TestGetFs(t *testing.T) {
defer mockNewFs(t)() defer mockNewFs(t)()