From a95c7a001edbad568e59c741945152adedd152a9 Mon Sep 17 00:00:00 2001 From: Ivan Andreev Date: Sat, 2 Oct 2021 12:51:00 +0300 Subject: [PATCH] core: run rclone as mount helper - #5594 --- fs/mount_helper.go | 286 +++++++++++++++++++++++++++++++++++ fs/mount_helper_test.go | 53 +++++++ lib/daemonize/daemon_unix.go | 52 +++++++ 3 files changed, 391 insertions(+) create mode 100644 fs/mount_helper.go create mode 100644 fs/mount_helper_test.go diff --git a/fs/mount_helper.go b/fs/mount_helper.go new file mode 100644 index 000000000..6a1a0953b --- /dev/null +++ b/fs/mount_helper.go @@ -0,0 +1,286 @@ +package fs + +import ( + "fmt" + "log" + "os" + "path/filepath" + "runtime" + "strings" + + "github.com/pkg/errors" +) + +func init() { + // This block is run super-early, before configuration harness kick in + if IsMountHelper() { + if args, err := convertMountHelperArgs(os.Args); err == nil { + os.Args = args + } else { + log.Fatalf("Failed to parse command line: %v", err) + } + } +} + +// PassDaemonArgsAsEnviron tells how CLI arguments are passed to the daemon +// When false, arguments are passed as is, visible in the `ps` output. +// When true, arguments are converted into environment variables (more secure). +var PassDaemonArgsAsEnviron bool + +// Comma-separated list of mount options to ignore. +// Leading and trailing commas are required. +const helperIgnoredOpts = ",rw,_netdev,nofail,user,dev,nodev,suid,nosuid,exec,noexec,auto,noauto," + +// Valid option name characters +const helperValidOptChars = "-_0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + +// Parser errors +var ( + errHelperBadOption = errors.New("option names may only contain `0-9`, `A-Z`, `a-z`, `-` and `_`") + errHelperOptionName = errors.New("option name can't start with `-` or `_`") + errHelperEmptyOption = errors.New("option name can't be empty") + errHelperQuotedValue = errors.New("unterminated quoted value") + errHelperAfterQuote = errors.New("expecting `,` or another quote after a quote") + errHelperSyntax = errors.New("syntax error in option string") + errHelperEmptyCommand = errors.New("command name can't be empty") + errHelperEnvSyntax = errors.New("environment variable must have syntax env.NAME=[VALUE]") +) + +// IsMountHelper returns true if rclone was invoked as mount helper: +// as /sbin/mount.rlone (by /bin/mount) +// or /usr/bin/rclonefs (by fusermount or directly) +func IsMountHelper() bool { + if runtime.GOOS == "windows" { + return false + } + me := filepath.Base(os.Args[0]) + return me == "mount.rclone" || me == "rclonefs" +} + +// convertMountHelperArgs converts "-o" styled mount helper arguments +// into usual rclone flags +func convertMountHelperArgs(origArgs []string) ([]string, error) { + if IsDaemon() { + // The arguments have already been converted by the parent + return origArgs, nil + } + + args := []string{} + command := "mount" + parseOpts := false + gotDaemon := false + gotVerbose := false + vCount := 0 + + for _, arg := range origArgs[1:] { + if !parseOpts { + switch arg { + case "-o", "--opt": + parseOpts = true + case "-v", "-vv", "-vvv", "-vvvv": + vCount += len(arg) - 1 + case "-h", "--help": + args = append(args, "--help") + default: + if strings.HasPrefix(arg, "-") { + return nil, errors.Errorf("flag %q is not supported in mount mode", arg) + } + args = append(args, arg) + } + continue + } + + opts, err := parseHelperOptionString(arg) + if err != nil { + return nil, err + } + parseOpts = false + + for _, opt := range opts { + if strings.Contains(helperIgnoredOpts, ","+opt+",") || strings.HasPrefix(opt, "x-systemd") { + continue + } + + param, value := opt, "" + if idx := strings.Index(opt, "="); idx != -1 { + param, value = opt[:idx], opt[idx+1:] + } + + // Set environment variables + if strings.HasPrefix(param, "env.") { + if param = param[4:]; param == "" { + return nil, errHelperEnvSyntax + } + _ = os.Setenv(param, value) + continue + } + + switch param { + // Change command to run + case "command": + if value == "" { + return nil, errHelperEmptyCommand + } + command = value + continue + // Flag StartDaemon to pass arguments as environment + case "args2env": + PassDaemonArgsAsEnviron = true + continue + // Handle verbosity options + case "v", "vv", "vvv", "vvvv": + vCount += len(param) + continue + case "verbose": + gotVerbose = true + // Don't add --daemon if it was explicitly included + case "daemon": + gotDaemon = true + // Alias for the standard mount option "ro" + case "ro": + param = "read-only" + } + + arg = "--" + strings.ToLower(strings.ReplaceAll(param, "_", "-")) + if value != "" { + arg += "=" + value + } + args = append(args, arg) + } + } + if parseOpts { + return nil, errors.Errorf("dangling -o without argument") + } + + if vCount > 0 && !gotVerbose { + args = append(args, fmt.Sprintf("--verbose=%d", vCount)) + } + if strings.Contains(command, "mount") && !gotDaemon { + // Default to daemonized mount + args = append(args, "--daemon") + } + if len(args) > 0 && args[0] == command { + // Remove artefact of repeated conversion + args = args[1:] + } + prepend := []string{origArgs[0], command} + return append(prepend, args...), nil +} + +// parseHelperOptionString deconstructs the -o value into slice of options +// in a way similar to connection strings. +// Example: +// param1=value,param2="qvalue",param3='item1,item2',param4="a ""b"" 'c'" +// An error may be returned if the remote name has invalid characters +// or the parameters are invalid or the path is empty. +// +// The algorithm was adapted from fspath.Parse with some modifications: +// - allow `-` in option names +// - handle special options `x-systemd.X` and `env.X` +// - drop support for :backend: and /path +func parseHelperOptionString(optString string) (opts []string, err error) { + if optString = strings.TrimSpace(optString); optString == "" { + return nil, nil + } + // States for parser + const ( + stateParam = uint8(iota) + stateValue + stateQuotedValue + stateAfterQuote + stateDone + ) + var ( + state = stateParam // current state of parser + i int // position in path + prev int // previous position in path + c rune // current rune under consideration + quote rune // kind of quote to end this quoted string + param string // current parameter value + doubled bool // set if had doubled quotes + ) + for i, c = range optString + "," { + switch state { + // Parses param= and param2= + case stateParam: + switch c { + case ',', '=': + param = optString[prev:i] + if len(param) == 0 { + return nil, errHelperEmptyOption + } + if param[0] == '-' || param[0] == '_' { + return nil, errHelperOptionName + } + prev = i + 1 + if c == '=' { + state = stateValue + break + } + opts = append(opts, param) + case '.': + if pref := optString[prev:i]; pref != "env" && pref != "x-systemd" { + return nil, errHelperBadOption + } + default: + if !strings.ContainsRune(helperValidOptChars, c) { + return nil, errHelperBadOption + } + } + case stateValue: + switch c { + case '\'', '"': + if i == prev { + quote = c + prev = i + 1 + doubled = false + state = stateQuotedValue + } + case ',': + value := optString[prev:i] + prev = i + 1 + opts = append(opts, param+"="+value) + state = stateParam + } + case stateQuotedValue: + if c == quote { + state = stateAfterQuote + } + case stateAfterQuote: + switch c { + case ',': + value := optString[prev : i-1] + // replace any doubled quotes if there were any + if doubled { + value = strings.ReplaceAll(value, string(quote)+string(quote), string(quote)) + } + prev = i + 1 + opts = append(opts, param+"="+value) + state = stateParam + case quote: + // Here is a doubled quote to indicate a literal quote + state = stateQuotedValue + doubled = true + default: + return nil, errHelperAfterQuote + } + } + } + + // Depending on which state we were in when we fell off the + // end of the state machine we can return a sensible error. + if state == stateParam && prev > len(optString) { + state = stateDone + } + switch state { + case stateQuotedValue: + return nil, errHelperQuotedValue + case stateAfterQuote: + return nil, errHelperAfterQuote + case stateDone: + break + default: + return nil, errHelperSyntax + } + return opts, nil +} diff --git a/fs/mount_helper_test.go b/fs/mount_helper_test.go new file mode 100644 index 000000000..00c4fba79 --- /dev/null +++ b/fs/mount_helper_test.go @@ -0,0 +1,53 @@ +package fs + +import ( + "os" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMountHelperArgs(t *testing.T) { + type testCase struct { + src []string + dst []string + env string + err string + } + normalCases := []testCase{{ + src: []string{}, + dst: []string{"mount", "--daemon"}, + }, { + src: []string{"-o", `x-systemd.automount,vvv,env.HTTPS_PROXY="a b;c,d?EF",ro,rw,args2env`}, + dst: []string{"mount", "--read-only", "--verbose=3", "--daemon"}, + env: "HTTPS_PROXY=a b;c,d?EF", + }} + + for _, tc := range normalCases { + exe := []string{"rclone"} + src := append(exe, tc.src...) + res, err := convertMountHelperArgs(src) + + if tc.err != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tc.err) + continue + } + + require.NoError(t, err) + require.Greater(t, len(res), 1) + assert.Equal(t, exe[0], res[0]) + dst := res[1:] + + //log.Printf("%q -> %q", tc.src, dst) + assert.Equal(t, tc.dst, dst) + + if tc.env != "" { + idx := strings.Index(tc.env, "=") + name, value := tc.env[:idx], tc.env[idx+1:] + assert.Equal(t, value, os.Getenv(name)) + } + } +} diff --git a/lib/daemonize/daemon_unix.go b/lib/daemonize/daemon_unix.go index 325ee3f80..beb89f7ac 100644 --- a/lib/daemonize/daemon_unix.go +++ b/lib/daemonize/daemon_unix.go @@ -7,6 +7,7 @@ package daemonize import ( "os" + "strings" "syscall" "github.com/rclone/rclone/fs" @@ -29,6 +30,18 @@ func StartDaemon(args []string) (*os.Process, error) { me = os.Args[0] } + // os.Executable might have resolved symbolic link to the executable + // so we run the background process with pre-converted CLI arguments. + // Double conversion is still probable but isn't a problem as it should + // preserve the converted command line. + if len(args) != 0 { + args[0] = me + } + + if fs.PassDaemonArgsAsEnviron { + args, env = argsToEnv(args, env) + } + null, err := os.Open(os.DevNull) if err != nil { return nil, err @@ -57,3 +70,42 @@ func StartDaemon(args []string) (*os.Process, error) { return daemon, nil } + +// Processed command line flags of mount helper have simple structure: +// `--flag` or `--flag=value` but never `--flag value` or `-x` +// so we can easily pass them as environment variables. +func argsToEnv(origArgs, origEnv []string) (args, env []string) { + env = origEnv + if len(origArgs) == 0 { + return + } + args = []string{origArgs[0]} + for _, arg := range origArgs[1:] { + if !strings.HasPrefix(arg, "--") { + args = append(args, arg) + continue + } + + arg = arg[2:] + key, val := arg, "true" + if idx := strings.Index(arg, "="); idx != -1 { + key, val = arg[:idx], arg[idx+1:] + } + + name := "RCLONE_" + strings.ToUpper(strings.ReplaceAll(key, "-", "_")) + + pref := name + "=" + line := name + "=" + val + found := false + for i, s := range env { + if strings.HasPrefix(s, pref) { + env[i] = line + found = true + } + } + if !found { + env = append(env, line) + } + } + return +}