diff --git a/bin/make_manual.py b/bin/make_manual.py index b7e040195..811cdbb64 100755 --- a/bin/make_manual.py +++ b/bin/make_manual.py @@ -18,6 +18,7 @@ docs = [ "docs.md", "remote_setup.md", "filtering.md", + "rc.md", "overview.md", # Keep these alphabetical by full name diff --git a/cmd/all/all.go b/cmd/all/all.go index 5351e320c..2b53370c2 100644 --- a/cmd/all/all.go +++ b/cmd/all/all.go @@ -36,6 +36,7 @@ import ( _ "github.com/ncw/rclone/cmd/ncdu" _ "github.com/ncw/rclone/cmd/obscure" _ "github.com/ncw/rclone/cmd/purge" + _ "github.com/ncw/rclone/cmd/rc" _ "github.com/ncw/rclone/cmd/rcat" _ "github.com/ncw/rclone/cmd/rmdir" _ "github.com/ncw/rclone/cmd/rmdirs" diff --git a/cmd/cmd.go b/cmd/cmd.go index 48b35a8a0..ae73d0417 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -30,6 +30,8 @@ import ( "github.com/ncw/rclone/fs/fserrors" "github.com/ncw/rclone/fs/fspath" fslog "github.com/ncw/rclone/fs/log" + "github.com/ncw/rclone/fs/rc" + "github.com/ncw/rclone/fs/rc/rcflags" "github.com/ncw/rclone/lib/atexit" ) @@ -126,6 +128,7 @@ func init() { // Add global flags configflags.AddFlags(pflag.CommandLine) filterflags.AddFlags(pflag.CommandLine) + rcflags.AddFlags(pflag.CommandLine) Root.Run = runRoot Root.Flags().BoolVarP(&version, "version", "V", false, "Print the version number") @@ -383,6 +386,9 @@ func initConfig() { // Write the args for debug purposes fs.Debugf("rclone", "Version %q starting with parameters %q", fs.Version, os.Args) + // Start the remote control if configured + rc.Start(&rcflags.Opt) + // Setup CPU profiling if desired if *cpuProfile != "" { fs.Infof(nil, "Creating CPU profile %q\n", *cpuProfile) diff --git a/cmd/rc/rc.go b/cmd/rc/rc.go new file mode 100644 index 000000000..cad7f2d77 --- /dev/null +++ b/cmd/rc/rc.go @@ -0,0 +1,139 @@ +package rc + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "os" + "strings" + + "github.com/ncw/rclone/cmd" + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/fshttp" + "github.com/ncw/rclone/fs/rc" + "github.com/pkg/errors" + "github.com/spf13/cobra" +) + +var ( + noOutput = false + url = "http://localhost:5572/" +) + +func init() { + cmd.Root.AddCommand(commandDefintion) + commandDefintion.Flags().BoolVarP(&noOutput, "no-output", "", noOutput, "If set don't output the JSON result.") + commandDefintion.Flags().StringVarP(&url, "url", "", url, "URL to connect to rclone remote control.") +} + +var commandDefintion = &cobra.Command{ + Use: "rc commands parameter", + Short: `Run a command against a running rclone.`, + Long: ` +This runs a command against a running rclone. By default it will use +that specified in the --rc-addr command. + +Arguments should be passed in as parameter=value. + +The result will be returned as a JSON object by default. + +Use "rclone rc list" to see a list of all possible commands.`, + Run: func(command *cobra.Command, args []string) { + cmd.CheckArgs(0, 1E9, command, args) + cmd.Run(false, false, command, func() error { + if len(args) == 0 { + return list() + } + return run(args) + }) + }, +} + +// do a call from (path, in) to (out, err). +// +// if err is set, out may be a valid error return or it may be nil +func doCall(path string, in rc.Params) (out rc.Params, err error) { + // Do HTTP request + client := fshttp.NewClient(fs.Config) + url := url + if !strings.HasSuffix(url, "/") { + url += "/" + } + url += path + data, err := json.Marshal(in) + if err != nil { + return nil, errors.Wrap(err, "failed to encode JSON") + } + resp, err := client.Post(url, "application/json", bytes.NewBuffer(data)) + if err != nil { + return nil, errors.Wrap(err, "connection failed") + } + defer fs.CheckClose(resp.Body, &err) + + // Parse output + out = make(rc.Params) + err = json.NewDecoder(resp.Body).Decode(&out) + if err != nil { + return nil, errors.Wrap(err, "failed to decode JSON") + } + + // Check we got 200 OK + if resp.StatusCode != http.StatusOK { + err = errors.Errorf("operation %q failed: %v", path, out["error"]) + } + + return out, err +} + +// Run the remote control command passed in +func run(args []string) (err error) { + path := strings.Trim(args[0], "/") + + // parse input + in := make(rc.Params) + for _, param := range args[1:] { + equals := strings.IndexRune(param, '=') + if equals < 0 { + return errors.Errorf("No '=' found in parameter %q", param) + } + key, value := param[:equals], param[equals+1:] + in[key] = value + } + + // Do the call + out, callErr := doCall(path, in) + + // Write the JSON blob to stdout if required + if out != nil && !noOutput { + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", "\t") + err = enc.Encode(out) + if err != nil { + return errors.Wrap(err, "failed to output JSON") + } + } + + return callErr +} + +// List the available commands to stdout +func list() error { + list, err := doCall("rc/list", nil) + if err != nil { + return errors.Wrap(err, "failed to list") + } + commands, ok := list["commands"].([]interface{}) + if !ok { + return errors.New("bad JSON") + } + for _, command := range commands { + info, ok := command.(map[string]interface{}) + if !ok { + return errors.New("bad JSON") + } + fmt.Printf("### %s: %s\n\n", info["Path"], info["Title"]) + fmt.Printf("%s\n\n", info["Help"]) + } + return nil +} diff --git a/docs/content/docs.md b/docs/content/docs.md index 4afa72cb0..43862002e 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -985,6 +985,16 @@ For the filtering options See the [filtering section](/filtering/). +Remote control +-------------- + +For the remote control options and for instructions on how to remote control rclone + + * `--rc` + * and anything starting with `--rc-` + +See [the remote control section](/rc/). + Logging ------- diff --git a/docs/content/rc.md b/docs/content/rc.md new file mode 100644 index 000000000..d0525064d --- /dev/null +++ b/docs/content/rc.md @@ -0,0 +1,145 @@ +--- +title: "Remote Control" +description: "Remote controlling rclone" +date: "2018-03-05" +--- + +# Remote controlling rclone # + +If rclone is run with the `--rc` flag then it starts an http server +which can be used to remote control rclone. + +FIXME describe other flags + +## Accessing the remote control via the rclone rc command + +Rclone itself implements the remote control protocol in its `rclone +rc` command. + +You can use it like this + + + +## Accessing the remote control via HTTP + +Rclone implements a simple HTTP based protocol. + +Each endpoint takes an JSON object and returns a JSON object or an +error. The JSON objects are essentially a map of string names to +values. + +All calls must made using POST. + +The input objects can be supplied using URL parameters, POST +parameters or by supplying "Content-Type: application/json" and a JSON +blob in the body. There are examples of these below using `curl`. + +The response will be a JSON blob in the body of the response. This is +formatted to be reasonably human readable. + +If an error occurs then there will be an HTTP error status (usually +400) and the body of the response will contain a JSON encoded error +object. + +### Using POST with URL parameters only + +``` +curl -X POST 'http://localhost:5572/rc/noop/?potato=1&sausage=2' +``` + +Response + +``` +{ + "potato": "1", + "sausage": "2" +} +``` + +Here is what an error response looks like: + +``` +curl -X POST 'http://localhost:5572/rc/error/?potato=1&sausage=2' +``` + +``` +{ + "error": "arbitrary error on input map[potato:1 sausage:2]", + "input": { + "potato": "1", + "sausage": "2" + } +} +``` + +Note that curl doesn't return errors to the shell unless you use the `-f` option + +``` +$ curl -f -X POST 'http://localhost:5572/rc/error/?potato=1&sausage=2' +curl: (22) The requested URL returned error: 400 Bad Request +$ echo $? +22 +``` + +### Using POST with a form + +``` +curl --data "potato=1" --data "sausage=2" http://localhost:5572/rc/noop/ +``` + +Response + +``` +{ + "potato": "1", + "sausage": "2" +} +``` + +Note that you can combine these with URL parameters too with the POST +parameters taking precedence. + +``` +curl --data "potato=1" --data "sausage=2" "http://localhost:5572/rc/noop/?rutabaga=3&sausage=4" +``` + +Response + +``` +{ + "potato": "1", + "rutabaga": "3", + "sausage": "4" +} + +``` + +### Using POST with a JSON blob + +``` +curl -H "Content-Type: application/json" -X POST -d '{"potato":2,"sausage":1}' http://localhost:5572/rc/noop/ +``` + +response + +``` +{ + "password": "xyz", + "username": "xyz" +} +``` + +This can be combined with URL parameters too if required. The JSON +blob takes precedence. + +``` +curl -H "Content-Type: application/json" -X POST -d '{"potato":2,"sausage":1}' 'http://localhost:5572/rc/noop/?rutabaga=3&potato=4' +``` + +``` +{ + "potato": 2, + "rutabaga": "3", + "sausage": 1 +} +``` diff --git a/docs/layouts/chrome/navbar.html b/docs/layouts/chrome/navbar.html index 2a40e1c02..19ddb4d3d 100644 --- a/docs/layouts/chrome/navbar.html +++ b/docs/layouts/chrome/navbar.html @@ -18,6 +18,7 @@
  • Installation
  • Usage
  • Filtering
    • +
  • Remote Control
  • Changelog
  • Bugs
  • FAQ
    • diff --git a/fs/rc/internal.go b/fs/rc/internal.go new file mode 100644 index 000000000..e462499a7 --- /dev/null +++ b/fs/rc/internal.go @@ -0,0 +1,50 @@ +// Define the internal rc functions + +package rc + +import "github.com/pkg/errors" + +func init() { + Add(Call{ + Path: "rc/noop", + Fn: rcNoop, + Title: "Echo the input to the output parameters", + Help: ` +This echoes the input parameters to the output parameters for testing +purposes. It can be used to check that rclone is still alive and to +check that parameter passing is working properly.`, + }) + Add(Call{ + Path: "rc/error", + Fn: rcError, + Title: "This returns an error", + Help: ` +This returns an error with the input as part of its error string. +Useful for testing error handling.`, + }) + Add(Call{ + Path: "rc/list", + Fn: rcList, + Title: "List all the registered remote control commands", + Help: ` +This lists all the registered remote control commands as a JSON map in +the commands response.`, + }) +} + +// Echo the input to the ouput parameters +func rcNoop(in Params) (out Params, err error) { + return in, nil +} + +// Return an error regardless +func rcError(in Params) (out Params, err error) { + return nil, errors.Errorf("arbitrary error on input %+v", in) +} + +// List the registered commands +func rcList(in Params) (out Params, err error) { + out = make(Params) + out["commands"] = registry.list() + return out, nil +} diff --git a/fs/rc/rc.go b/fs/rc/rc.go new file mode 100644 index 000000000..a0d3b614d --- /dev/null +++ b/fs/rc/rc.go @@ -0,0 +1,135 @@ +// Package rc implements a remote control server and registry for rclone +// +// To register your internal calls, call rc.Add(path, function). Your +// function should take ane return a Param. It can also return an +// error. Use rc.NewError to wrap an existing error along with an +// http response type if another response other than 500 internal +// error is required on error. +package rc + +import ( + "encoding/json" + "net/http" + "strings" + + "github.com/ncw/rclone/cmd/serve/httplib" + "github.com/ncw/rclone/fs" + "github.com/pkg/errors" +) + +// Options contains options for the remote control server +type Options struct { + HTTPOptions httplib.Options + Enabled bool +} + +// DefaultOpt is the default values used for Options +var DefaultOpt = Options{ + HTTPOptions: httplib.DefaultOpt, + Enabled: false, +} + +func init() { + DefaultOpt.HTTPOptions.ListenAddr = "localhost:5572" +} + +// Start the remote control server if configured +func Start(opt *Options) { + if opt.Enabled { + s := newServer(opt) + go s.serve() + } +} + +// server contains everything to run the server +type server struct { + srv *httplib.Server +} + +func newServer(opt *Options) *server { + // Serve on the DefaultServeMux so can have global registrations appear + mux := http.DefaultServeMux + s := &server{ + srv: httplib.NewServer(mux, &opt.HTTPOptions), + } + mux.HandleFunc("/", s.handler) + return s +} + +// serve runs the http server - doesn't return +func (s *server) serve() { + fs.Logf(nil, "Serving remote control on %s", s.srv.URL()) + s.srv.Serve() +} + +// writes JSON in out to w +func writeJSON(w http.ResponseWriter, out Params) { + enc := json.NewEncoder(w) + enc.SetIndent("", "\t") + err := enc.Encode(out) + if err != nil { + // can't return the error at this point + fs.Errorf(nil, "rc: failed to write JSON output: %v", err) + } +} + +// handler reads incoming requests and dispatches them +func (s *server) handler(w http.ResponseWriter, r *http.Request) { + path := strings.Trim(r.URL.Path, "/") + in := make(Params) + + writeError := func(err error, status int) { + fs.Errorf(nil, "rc: %q: error: %v", path, err) + w.WriteHeader(status) + writeJSON(w, Params{ + "error": err.Error(), + "input": in, + }) + } + + if r.Method != "POST" { + writeError(errors.Errorf("method %q not allowed - POST required", r.Method), http.StatusMethodNotAllowed) + return + } + + // Find the call + call := registry.get(path) + if call == nil { + writeError(errors.Errorf("couldn't find method %q", path), http.StatusMethodNotAllowed) + return + } + + // Parse the POST and URL parameters into r.Form + err := r.ParseForm() + if err != nil { + writeError(errors.Wrap(err, "failed to parse form/URL parameters"), http.StatusBadRequest) + return + } + + // Read the POST and URL parameters into in + for k, vs := range r.Form { + if len(vs) > 0 { + in[k] = vs[len(vs)-1] + } + } + fs.Debugf(nil, "form = %+v", r.Form) + + // Parse a JSON blob from the input + if r.Header.Get("Content-Type") == "application/json" { + err := json.NewDecoder(r.Body).Decode(&in) + if err != nil { + writeError(errors.Wrap(err, "failed to read input JSON"), http.StatusBadRequest) + return + } + } + + fs.Debugf(nil, "rc: %q: with parameters %+v", path, in) + out, err := call.Fn(in) + if err != nil { + writeError(errors.Wrap(err, "remote control command failed"), http.StatusInternalServerError) + return + } + + fs.Debugf(nil, "rc: %q: reply %+v: %v", path, out, err) + writeJSON(w, out) +} diff --git a/fs/rc/rcflags/rcflags.go b/fs/rc/rcflags/rcflags.go new file mode 100644 index 000000000..a8f921a9a --- /dev/null +++ b/fs/rc/rcflags/rcflags.go @@ -0,0 +1,20 @@ +// Package rcflags implements command line flags to set up the remote control +package rcflags + +import ( + "github.com/ncw/rclone/cmd/serve/httplib/httpflags" + "github.com/ncw/rclone/fs/config/flags" + "github.com/ncw/rclone/fs/rc" + "github.com/spf13/pflag" +) + +// Options set by command line flags +var ( + Opt = rc.DefaultOpt +) + +// AddFlags adds the remote control flags to the flagSet +func AddFlags(flagSet *pflag.FlagSet) { + flags.BoolVarP(flagSet, &Opt.Enabled, "rc", "", false, "Enable the remote control server.") + httpflags.AddFlagsPrefix(flagSet, "rc-", &Opt.HTTPOptions) +} diff --git a/fs/rc/registry.go b/fs/rc/registry.go new file mode 100644 index 000000000..9287805bd --- /dev/null +++ b/fs/rc/registry.go @@ -0,0 +1,73 @@ +// Define the registry + +package rc + +import ( + "strings" + "sync" + + "github.com/ncw/rclone/fs" +) + +// Params is the input and output type for the Func +type Params map[string]interface{} + +// Func defines a type for a remote control function +type Func func(in Params) (out Params, err error) + +// Call defines info about a remote control function and is used in +// the Add function to create new entry points. +type Call struct { + Path string // path to activate this RC + Fn Func `json:"-"` // function to call + Title string // help for the function + Help string // multi-line markdown formatted help +} + +// Registry holds the list of all the registered remote control functions +type Registry struct { + mu sync.RWMutex + call map[string]*Call +} + +// NewRegistry makes a new registry for remote control functions +func NewRegistry() *Registry { + return &Registry{ + call: make(map[string]*Call), + } +} + +// Add a call to the registry +func (r *Registry) add(call Call) { + r.mu.Lock() + defer r.mu.Unlock() + call.Path = strings.Trim(call.Path, "/") + call.Help = strings.TrimSpace(call.Help) + fs.Debugf(nil, "Adding path %q to remote control registry", call.Path) + r.call[call.Path] = &call +} + +// get a Call from a path or nil +func (r *Registry) get(path string) *Call { + r.mu.RLock() + defer r.mu.RUnlock() + return r.call[path] +} + +// get a list of all calls +func (r *Registry) list() (out []*Call) { + r.mu.RLock() + defer r.mu.RUnlock() + for _, call := range r.call { + out = append(out, call) + } + return out +} + +// The global registry +var registry = NewRegistry() + +// Add a function to the global registry +func Add(call Call) { + registry.add(call) +}