community
/
rclone
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

backend: add new backend command for backend specific commands 

These commands are for implementing backend specific
functionality. They have documentation which is placed automatically
into the backend doc.

There is a simple test for the feature in the backend tests.
This commit is contained in:
Nick Craig-Wood 2020-04-28 12:58:34 +01:00
parent 195d152785
commit 1aa1a2c174
5 changed files with 242 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
bin/make_backend_docs.py
View File

@ -4,6 +4,7 @@ Make backend documentation
"""
import os
import io
import subprocess
marker = "<!--- autogenerated options"
@ -19,6 +20,11 @@ def output_docs(backend, out):
out.flush()
subprocess.check_call(["rclone", "help", "backend", backend], stdout=out)
def output_backend_tool_docs(backend, out):
"""Output documentation for backend tool to out"""
out.flush()
subprocess.call(["rclone", "backend", "help", backend], stdout=out, stderr=subprocess.DEVNULL)
def alter_doc(backend):
"""Alter the documentation for backend"""
doc_file = "docs/content/"+backend+".md"
@ -35,6 +41,7 @@ def alter_doc(backend):
start_full = start + " - DO NOT EDIT, instead edit fs.RegInfo in backend/%s/%s.go then run make backenddocs -->\n" % (backend, backend)
out_file.write(start_full)
output_docs(backend, out_file)
output_backend_tool_docs(backend, out_file)
out_file.write(stop+" -->\n")
altered = True
if not in_docs:

1
cmd/all/all.go
View File

@ -6,6 +6,7 @@ import (
_ "github.com/rclone/rclone/cmd"
_ "github.com/rclone/rclone/cmd/about"
_ "github.com/rclone/rclone/cmd/authorize"
_ "github.com/rclone/rclone/cmd/backend"
_ "github.com/rclone/rclone/cmd/cachestats"
_ "github.com/rclone/rclone/cmd/cat"
_ "github.com/rclone/rclone/cmd/check"

169
cmd/backend/backend.go Normal file
View File

@ -0,0 +1,169 @@
package backend
import (
"context"
"encoding/json"
"fmt"
"os"
"sort"
"github.com/pkg/errors"
"github.com/rclone/rclone/cmd"
"github.com/rclone/rclone/cmd/rc"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config/flags"
"github.com/rclone/rclone/fs/operations"
"github.com/spf13/cobra"
)
var (
options []string
)
func init() {
cmd.Root.AddCommand(commandDefinition)
cmdFlags := commandDefinition.Flags()
flags.StringArrayVarP(cmdFlags, &options, "option", "o", options, "Option in the form name=value or name.")
}
var commandDefinition = &cobra.Command{
Use: "backend <command> remote:path [opts] <args>",
Short: `Run a backend specific command.`,
Long: `
This runs a backend specific command. The commands themselves (except
for "help" and "features") are defined by the backends and you should
see the backend docs for definitions.
You can discover what commands a backend implements by using
rclone backend help remote:
rclone backend help <backendname>
You can also discover information about the backend using (see
[operations/fsinfo](/rc/#operations/fsinfo) in the remote control docs
for more info).
rclone backend features remote:
Pass options to the backend command with -o. This should be key=value or key, eg:
rclone backend stats remote:path stats -o format=json -o long
Pass arguments to the backend by placing them on the end of the line
rclone backend cleanup remote:path file1 file2 file3
Note to run these commands on a running backend then see
[backend/command](/rc/#backend/command) in the rc docs.
`,
RunE: func(command *cobra.Command, args []string) error {
cmd.CheckArgs(2, 1E6, command, args)
name, remote := args[0], args[1]
cmd.Run(false, false, command, func() error {
// show help if remote is a backend name
if name == "help" {
fsInfo, err := fs.Find(remote)
if err == nil {
return showHelp(fsInfo)
}
}
// Create remote
fsInfo, configName, fsPath, config, err := fs.ConfigFs(remote)
if err != nil {
return err
}
f, err := fsInfo.NewFs(configName, fsPath, config)
if err != nil {
return err
}
// Run the command
var out interface{}
switch name {
case "help":
return showHelp(fsInfo)
case "features":
out = operations.GetFsInfo(f)
default:
doCommand := f.Features().Command
if doCommand == nil {
return errors.Errorf("%v: doesn't support backend commands", f)
}
arg := args[2:]
opt := rc.ParseOptions(options)
out, err = doCommand(context.Background(), name, arg, opt)
}
if err != nil {
return errors.Wrapf(err, "command %q failed", name)
}
// Output the result
switch x := out.(type) {
case nil:
case string:
fmt.Println(out)
case []string:
for line := range x {
fmt.Println(line)
}
default:
// Write indented JSON to the output
enc := json.NewEncoder(os.Stdout)
enc.SetIndent("", "\t")
err = enc.Encode(out)
if err != nil {
return errors.Wrap(err, "failed to write JSON")
}
}
return nil
})
return nil
},
}
// show help for a backend
func showHelp(fsInfo *fs.RegInfo) error {
cmds := fsInfo.CommandHelp
name := fsInfo.Name
if len(cmds) == 0 {
return errors.Errorf("%s backend has no commands", name)
}
fmt.Printf("### Backend commands\n\n")
fmt.Printf(`Here are the commands specific to the %s backend.
Run them with with
rclone backend COMMAND remote:
The help below will explain what arguments each command takes.
See [the "rclone backend" command](/commands/rclone_backend/) for more
info on how to pass options and arguments.
These can be run on a running backend using the rc command
[backend/command](/rc/#backend/command).
`, name)
for _, cmd := range cmds {
fmt.Printf("#### %s\n\n", cmd.Name)
fmt.Printf("%s\n\n", cmd.Short)
fmt.Printf(" rclone backend %s remote: [options] [<arguments>+]\n\n", cmd.Name)
if cmd.Long != "" {
fmt.Printf("%s\n\n", cmd.Long)
}
if len(cmd.Opts) != 0 {
fmt.Printf("Options:\n\n")
ks := []string{}
for k := range cmd.Opts {
ks = append(ks, k)
}
sort.Strings(ks)
for _, k := range ks {
v := cmd.Opts[k]
fmt.Printf("- %q: %s\n", k, v)
}
fmt.Printf("\n")
}
}
return nil
}

42
fs/fs.go
View File

@ -70,6 +70,7 @@ var (
ErrorPermissionDenied = errors.New("permission denied")
ErrorCantShareDirectories = errors.New("this backend can't share directories with link")
ErrorNotImplemented = errors.New("optional feature not implemented")
ErrorCommandNotFound = errors.New("command not found")
)
// RegInfo provides information about a filesystem
@ -88,6 +89,8 @@ type RegInfo struct {
Config func(name string, config configmap.Mapper) `json:"-"`
// Options for the Fs configuration
Options Options
// The command help, if any
CommandHelp []CommandHelp
}
// FileName returns the on disk file name for this backend
@ -634,6 +637,17 @@ type Features struct {
// Disconnect the current user
Disconnect func(ctx context.Context) error
// Command the backend to run a named command
//
// The command run is name
// args may be used to read arguments from
// opts may be used to read optional arguments from
//
// The result should be capable of being JSON encoded
// If it is a string or a []string it will be shown to the user
// otherwise it will be JSON encoded and shown to the user like that
Command func(ctx context.Context, name string, arg []string, opt map[string]string) (interface{}, error)
}
// Disable nil's out the named feature. If it isn't found then it
@ -755,6 +769,9 @@ func (ft *Features) Fill(f Fs) *Features {
if do, ok := f.(Disconnecter); ok {
ft.Disconnect = do.Disconnect
}
if do, ok := f.(Commander); ok {
ft.Command = do.Command
}
return ft.DisableList(Config.DisableFeatures)
}
@ -830,6 +847,7 @@ func (ft *Features) Mask(f Fs) *Features {
if mask.Disconnect == nil {
ft.Disconnect = nil
}
// Command is always local so we don't mask it
return ft.DisableList(Config.DisableFeatures)
}
@ -1051,6 +1069,30 @@ type Disconnecter interface {
Disconnect(ctx context.Context) error
}
// CommandHelp describes a single backend Command
//
// These are automatically inserted in the docs
type CommandHelp struct {
Name string // Name of the command, eg "link"
Short string // Single line description
Long string // Long multi-line description
Opts map[string]string // maps option name to a single line help
}
// Commander is an iterface to wrap the Command function
type Commander interface {
// Command the backend to run a named command
//
// The command run is name
// args may be used to read arguments from
// opts may be used to read optional arguments from
//
// The result should be capable of being JSON encoded
// If it is a string or a []string it will be shown to the user
// otherwise it will be JSON encoded and shown to the user like that
Command(ctx context.Context, name string, arg []string, opt map[string]string) (interface{}, error)
}
// ObjectsChan is a channel of Objects
type ObjectsChan chan Object

26
fstest/fstests/fstests.go
View File

@ -292,9 +292,10 @@ func Run(t *testing.T, opt *Opt) {
ModTime: fstest.Time("2001-02-03T04:05:10.123123123Z"),
Path: `hello? sausage/êé/Hello, 世界/ " ' @ < > & ? + ≠/z.txt`,
}
isLocalRemote bool
purged bool // whether the dir has been purged or not
ctx = context.Background()
isLocalRemote bool
purged bool // whether the dir has been purged or not
ctx = context.Background()
unwrappableFsMethods = []string{"Command"} // these Fs methods don't need to be wrapped ever
)
if strings.HasSuffix(os.Getenv("RCLONE_CONFIG"), "/notfound") && *fstest.RemoteName == "" {
@ -398,6 +399,9 @@ func Run(t *testing.T, opt *Opt) {
if stringsContains(vName, opt.UnimplementableFsMethods) {
continue
}
if stringsContains(vName, unwrappableFsMethods) {
continue
}
field := v.Field(i)
// skip the bools
if field.Type().Kind() == reflect.Bool {
@ -409,6 +413,22 @@ func Run(t *testing.T, opt *Opt) {
}
})
// Check to see if Fs advertises commands and they work and have docs
t.Run("FsCommand", func(t *testing.T) {
skipIfNotOk(t)
doCommand := remote.Features().Command
if doCommand == nil {
t.Skip("No commands in this remote")
}
// Check the correct error is generated
_, err := doCommand(context.Background(), "NOTFOUND", nil, nil)
assert.Equal(t, fs.ErrorCommandNotFound, err, "Incorrect error generated on command not found")
// Check there are some commands in the fsInfo
fsInfo, _, _, _, err := fs.ConfigFs(remoteName)
require.NoError(t, err)
assert.True(t, len(fsInfo.CommandHelp) > 0, "Command is declared, must return some help in CommandHelp")
})
// TestFsRmdirNotFound tests deleting a non existent directory
t.Run("FsRmdirNotFound", func(t *testing.T) {
skipIfNotOk(t)