From 1aa1a2c174cdd598450521249c88677b83974e08 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 28 Apr 2020 12:58:34 +0100 Subject: [PATCH] backend: add new backend command for backend specific commands These commands are for implementing backend specific functionality. They have documentation which is placed automatically into the backend doc. There is a simple test for the feature in the backend tests. --- bin/make_backend_docs.py | 7 ++ cmd/all/all.go | 1 + cmd/backend/backend.go | 169 ++++++++++++++++++++++++++++++++++++++ fs/fs.go | 42 ++++++++++ fstest/fstests/fstests.go | 26 +++++- 5 files changed, 242 insertions(+), 3 deletions(-) create mode 100644 cmd/backend/backend.go diff --git a/bin/make_backend_docs.py b/bin/make_backend_docs.py index a30df22ad..b3df88808 100755 --- a/bin/make_backend_docs.py +++ b/bin/make_backend_docs.py @@ -4,6 +4,7 @@ Make backend documentation """ import os +import io import subprocess marker = "\n" % (backend, backend) out_file.write(start_full) output_docs(backend, out_file) + output_backend_tool_docs(backend, out_file) out_file.write(stop+" -->\n") altered = True if not in_docs: diff --git a/cmd/all/all.go b/cmd/all/all.go index 894e04b1c..c3ad8bc78 100644 --- a/cmd/all/all.go +++ b/cmd/all/all.go @@ -6,6 +6,7 @@ import ( _ "github.com/rclone/rclone/cmd" _ "github.com/rclone/rclone/cmd/about" _ "github.com/rclone/rclone/cmd/authorize" + _ "github.com/rclone/rclone/cmd/backend" _ "github.com/rclone/rclone/cmd/cachestats" _ "github.com/rclone/rclone/cmd/cat" _ "github.com/rclone/rclone/cmd/check" diff --git a/cmd/backend/backend.go b/cmd/backend/backend.go new file mode 100644 index 000000000..073cc1150 --- /dev/null +++ b/cmd/backend/backend.go @@ -0,0 +1,169 @@ +package backend + +import ( + "context" + "encoding/json" + "fmt" + "os" + "sort" + + "github.com/pkg/errors" + "github.com/rclone/rclone/cmd" + "github.com/rclone/rclone/cmd/rc" + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/config/flags" + "github.com/rclone/rclone/fs/operations" + "github.com/spf13/cobra" +) + +var ( + options []string +) + +func init() { + cmd.Root.AddCommand(commandDefinition) + cmdFlags := commandDefinition.Flags() + flags.StringArrayVarP(cmdFlags, &options, "option", "o", options, "Option in the form name=value or name.") +} + +var commandDefinition = &cobra.Command{ + Use: "backend remote:path [opts] ", + Short: `Run a backend specific command.`, + Long: ` +This runs a backend specific command. The commands themselves (except +for "help" and "features") are defined by the backends and you should +see the backend docs for definitions. + +You can discover what commands a backend implements by using + + rclone backend help remote: + rclone backend help + +You can also discover information about the backend using (see +[operations/fsinfo](/rc/#operations/fsinfo) in the remote control docs +for more info). + + rclone backend features remote: + +Pass options to the backend command with -o. This should be key=value or key, eg: + + rclone backend stats remote:path stats -o format=json -o long + +Pass arguments to the backend by placing them on the end of the line + + rclone backend cleanup remote:path file1 file2 file3 + +Note to run these commands on a running backend then see +[backend/command](/rc/#backend/command) in the rc docs. +`, + RunE: func(command *cobra.Command, args []string) error { + cmd.CheckArgs(2, 1E6, command, args) + name, remote := args[0], args[1] + cmd.Run(false, false, command, func() error { + // show help if remote is a backend name + if name == "help" { + fsInfo, err := fs.Find(remote) + if err == nil { + return showHelp(fsInfo) + } + } + // Create remote + fsInfo, configName, fsPath, config, err := fs.ConfigFs(remote) + if err != nil { + return err + } + f, err := fsInfo.NewFs(configName, fsPath, config) + if err != nil { + return err + } + // Run the command + var out interface{} + switch name { + case "help": + return showHelp(fsInfo) + case "features": + out = operations.GetFsInfo(f) + default: + doCommand := f.Features().Command + if doCommand == nil { + return errors.Errorf("%v: doesn't support backend commands", f) + } + arg := args[2:] + opt := rc.ParseOptions(options) + out, err = doCommand(context.Background(), name, arg, opt) + } + if err != nil { + return errors.Wrapf(err, "command %q failed", name) + + } + // Output the result + switch x := out.(type) { + case nil: + case string: + fmt.Println(out) + case []string: + for line := range x { + fmt.Println(line) + } + default: + // Write indented JSON to the output + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", "\t") + err = enc.Encode(out) + if err != nil { + return errors.Wrap(err, "failed to write JSON") + } + } + return nil + }) + return nil + }, +} + +// show help for a backend +func showHelp(fsInfo *fs.RegInfo) error { + cmds := fsInfo.CommandHelp + name := fsInfo.Name + if len(cmds) == 0 { + return errors.Errorf("%s backend has no commands", name) + } + fmt.Printf("### Backend commands\n\n") + fmt.Printf(`Here are the commands specific to the %s backend. + +Run them with with + + rclone backend COMMAND remote: + +The help below will explain what arguments each command takes. + +See [the "rclone backend" command](/commands/rclone_backend/) for more +info on how to pass options and arguments. + +These can be run on a running backend using the rc command +[backend/command](/rc/#backend/command). + +`, name) + for _, cmd := range cmds { + fmt.Printf("#### %s\n\n", cmd.Name) + fmt.Printf("%s\n\n", cmd.Short) + fmt.Printf(" rclone backend %s remote: [options] [+]\n\n", cmd.Name) + if cmd.Long != "" { + fmt.Printf("%s\n\n", cmd.Long) + } + if len(cmd.Opts) != 0 { + fmt.Printf("Options:\n\n") + + ks := []string{} + for k := range cmd.Opts { + ks = append(ks, k) + } + sort.Strings(ks) + for _, k := range ks { + v := cmd.Opts[k] + fmt.Printf("- %q: %s\n", k, v) + } + fmt.Printf("\n") + } + } + return nil +} diff --git a/fs/fs.go b/fs/fs.go index d8db0c255..2e4a18d8e 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -70,6 +70,7 @@ var ( ErrorPermissionDenied = errors.New("permission denied") ErrorCantShareDirectories = errors.New("this backend can't share directories with link") ErrorNotImplemented = errors.New("optional feature not implemented") + ErrorCommandNotFound = errors.New("command not found") ) // RegInfo provides information about a filesystem @@ -88,6 +89,8 @@ type RegInfo struct { Config func(name string, config configmap.Mapper) `json:"-"` // Options for the Fs configuration Options Options + // The command help, if any + CommandHelp []CommandHelp } // FileName returns the on disk file name for this backend @@ -634,6 +637,17 @@ type Features struct { // Disconnect the current user Disconnect func(ctx context.Context) error + + // Command the backend to run a named command + // + // The command run is name + // args may be used to read arguments from + // opts may be used to read optional arguments from + // + // The result should be capable of being JSON encoded + // If it is a string or a []string it will be shown to the user + // otherwise it will be JSON encoded and shown to the user like that + Command func(ctx context.Context, name string, arg []string, opt map[string]string) (interface{}, error) } // Disable nil's out the named feature. If it isn't found then it @@ -755,6 +769,9 @@ func (ft *Features) Fill(f Fs) *Features { if do, ok := f.(Disconnecter); ok { ft.Disconnect = do.Disconnect } + if do, ok := f.(Commander); ok { + ft.Command = do.Command + } return ft.DisableList(Config.DisableFeatures) } @@ -830,6 +847,7 @@ func (ft *Features) Mask(f Fs) *Features { if mask.Disconnect == nil { ft.Disconnect = nil } + // Command is always local so we don't mask it return ft.DisableList(Config.DisableFeatures) } @@ -1051,6 +1069,30 @@ type Disconnecter interface { Disconnect(ctx context.Context) error } +// CommandHelp describes a single backend Command +// +// These are automatically inserted in the docs +type CommandHelp struct { + Name string // Name of the command, eg "link" + Short string // Single line description + Long string // Long multi-line description + Opts map[string]string // maps option name to a single line help +} + +// Commander is an iterface to wrap the Command function +type Commander interface { + // Command the backend to run a named command + // + // The command run is name + // args may be used to read arguments from + // opts may be used to read optional arguments from + // + // The result should be capable of being JSON encoded + // If it is a string or a []string it will be shown to the user + // otherwise it will be JSON encoded and shown to the user like that + Command(ctx context.Context, name string, arg []string, opt map[string]string) (interface{}, error) +} + // ObjectsChan is a channel of Objects type ObjectsChan chan Object diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go index f5da52cc6..d082df5f7 100644 --- a/fstest/fstests/fstests.go +++ b/fstest/fstests/fstests.go @@ -292,9 +292,10 @@ func Run(t *testing.T, opt *Opt) { ModTime: fstest.Time("2001-02-03T04:05:10.123123123Z"), Path: `hello? sausage/êé/Hello, 世界/ " ' @ < > & ? + ≠/z.txt`, } - isLocalRemote bool - purged bool // whether the dir has been purged or not - ctx = context.Background() + isLocalRemote bool + purged bool // whether the dir has been purged or not + ctx = context.Background() + unwrappableFsMethods = []string{"Command"} // these Fs methods don't need to be wrapped ever ) if strings.HasSuffix(os.Getenv("RCLONE_CONFIG"), "/notfound") && *fstest.RemoteName == "" { @@ -398,6 +399,9 @@ func Run(t *testing.T, opt *Opt) { if stringsContains(vName, opt.UnimplementableFsMethods) { continue } + if stringsContains(vName, unwrappableFsMethods) { + continue + } field := v.Field(i) // skip the bools if field.Type().Kind() == reflect.Bool { @@ -409,6 +413,22 @@ func Run(t *testing.T, opt *Opt) { } }) + // Check to see if Fs advertises commands and they work and have docs + t.Run("FsCommand", func(t *testing.T) { + skipIfNotOk(t) + doCommand := remote.Features().Command + if doCommand == nil { + t.Skip("No commands in this remote") + } + // Check the correct error is generated + _, err := doCommand(context.Background(), "NOTFOUND", nil, nil) + assert.Equal(t, fs.ErrorCommandNotFound, err, "Incorrect error generated on command not found") + // Check there are some commands in the fsInfo + fsInfo, _, _, _, err := fs.ConfigFs(remoteName) + require.NoError(t, err) + assert.True(t, len(fsInfo.CommandHelp) > 0, "Command is declared, must return some help in CommandHelp") + }) + // TestFsRmdirNotFound tests deleting a non existent directory t.Run("FsRmdirNotFound", func(t *testing.T) { skipIfNotOk(t)