diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go index 9f7de8725..a75d91902 100644 --- a/backend/sftp/sftp.go +++ b/backend/sftp/sftp.go @@ -39,6 +39,8 @@ import ( ) const ( + defaultShellType = "unix" + shellTypeNotSupported = "none" hashCommandNotSupported = "none" minSleep = 100 * time.Millisecond maxSleep = 2 * time.Second @@ -47,7 +49,9 @@ const ( ) var ( - currentUser = env.CurrentUser() + currentUser = env.CurrentUser() + posixWinAbsPathRegex = regexp.MustCompile(`^/[a-zA-Z]\:($|/)`) // E.g. "/C:" or anything starting with "/C:/" + unixShellEscapeRegex = regexp.MustCompile("[^A-Za-z0-9_.,:/\\@\u0080-\uFFFFFFFF\n-]") ) func init() { @@ -148,16 +152,16 @@ If this is set and no password is supplied then rclone will: }, { Name: "path_override", Default: "", - Help: `Override path used by SSH connection. + Help: `Override path used by SSH shell commands. This allows checksum calculation when SFTP and SSH paths are different. This issue affects among others Synology NAS boxes. -Shared folders can be found in directories representing volumes +E.g. if shared folders can be found in directories representing volumes: rclone sync /home/local/directory remote:/directory --sftp-path-override /volume2/directory -Home directory can be found in a shared folder called "home" +E.g. if home directory can be found in a shared folder called "home": rclone sync /home/local/directory remote:/home/directory --sftp-path-override /volume1/homes/USER/directory`, Advanced: true, @@ -166,6 +170,26 @@ Home directory can be found in a shared folder called "home" Default: true, Help: "Set the modified time on the remote if set.", Advanced: true, + }, { + Name: "shell_type", + Default: "", + Help: "The type of SSH shell on remote server, if any.\n\nLeave blank for autodetect.", + Advanced: true, + Examples: []fs.OptionExample{ + { + Value: shellTypeNotSupported, + Help: "No shell access", + }, { + Value: "unix", + Help: "Unix shell", + }, { + Value: "powershell", + Help: "PowerShell", + }, { + Value: "cmd", + Help: "Windows Command Prompt", + }, + }, }, { Name: "md5sum_command", Default: "", @@ -270,6 +294,7 @@ type Options struct { AskPassword bool `config:"ask_password"` PathOverride string `config:"path_override"` SetModTime bool `config:"set_modtime"` + ShellType string `config:"shell_type"` Md5sumCommand string `config:"md5sum_command"` Sha1sumCommand string `config:"sha1sum_command"` SkipLinks bool `config:"skip_links"` @@ -286,6 +311,8 @@ type Fs struct { name string root string absRoot string + shellRoot string + shellType string opt Options // parsed options ci *fs.ConfigInfo // global config m configmap.Mapper // config @@ -542,7 +569,7 @@ func (f *Fs) drainPool(ctx context.Context) (err error) { f.drain.Stop() } if len(f.pool) != 0 { - fs.Debugf(f, "closing %d unused connections", len(f.pool)) + fs.Debugf(f, "Closing %d unused connections", len(f.pool)) } for i, c := range f.pool { if cErr := c.closed(); cErr == nil { @@ -739,7 +766,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e // // Just send the password back for all questions func (f *Fs) keyboardInteractiveReponse(user, instruction string, questions []string, echos []bool, pass string) ([]string, error) { - fs.Debugf(f, "keyboard interactive auth requested") + fs.Debugf(f, "Keyboard interactive auth requested") answers := make([]string, len(questions)) for i := range answers { answers[i] = pass @@ -769,6 +796,7 @@ func NewFsWithConnection(ctx context.Context, f *Fs, name string, root string, m f.name = name f.root = root f.absRoot = root + f.shellRoot = root f.opt = *opt f.m = m f.config = sshConfig @@ -778,7 +806,7 @@ func NewFsWithConnection(ctx context.Context, f *Fs, name string, root string, m f.savedpswd = "" // set the pool drainer timer going if f.opt.IdleTimeout > 0 { - f.drain = time.AfterFunc(time.Duration(opt.IdleTimeout), func() { _ = f.drainPool(ctx) }) + f.drain = time.AfterFunc(time.Duration(f.opt.IdleTimeout), func() { _ = f.drainPool(ctx) }) } f.features = (&fs.Features{ @@ -790,16 +818,59 @@ func NewFsWithConnection(ctx context.Context, f *Fs, name string, root string, m if err != nil { return nil, fmt.Errorf("NewFs: %w", err) } - cwd, err := c.sftpClient.Getwd() - f.putSftpConnection(&c, nil) - if err != nil { - fs.Debugf(f, "Failed to read current directory - using relative paths: %v", err) - } else if !path.IsAbs(f.root) { - f.absRoot = path.Join(cwd, f.root) - fs.Debugf(f, "Using absolute root directory %q", f.absRoot) + // Check remote shell type, try to auto-detect if not configured and save to config for later + if f.opt.ShellType != "" { + f.shellType = f.opt.ShellType + fs.Debugf(f, "Shell type %q from config", f.shellType) + } else { + session, err := c.sshClient.NewSession() + if err != nil { + f.shellType = shellTypeNotSupported + fs.Debugf(f, "Failed to get shell session for shell type detection command: %v", err) + } else { + var stdout, stderr bytes.Buffer + session.Stdout = &stdout + session.Stderr = &stderr + shellCmd := "echo ${ShellId}%ComSpec%" + fs.Debugf(f, "Running shell type detection remote command: %s", shellCmd) + err = session.Run(shellCmd) + _ = session.Close() + if err != nil { + f.shellType = defaultShellType + fs.Debugf(f, "Remote command failed: %v (stdout=%v) (stderr=%v)", err, bytes.TrimSpace(stdout.Bytes()), bytes.TrimSpace(stderr.Bytes())) + } else { + outBytes := stdout.Bytes() + fs.Debugf(f, "Remote command result: %s", outBytes) + outString := string(bytes.TrimSpace(stdout.Bytes())) + if strings.HasPrefix(outString, "Microsoft.PowerShell") { // If PowerShell: "Microsoft.PowerShell%ComSpec%" + f.shellType = "powershell" + } else if !strings.HasSuffix(outString, "%ComSpec%") { // If Command Prompt: "${ShellId}C:\WINDOWS\system32\cmd.exe" + f.shellType = "cmd" + } else { // If Unix: "%ComSpec%" + f.shellType = "unix" + } + } + } + // Save permanently in config to avoid the extra work next time + fs.Debugf(f, "Shell type %q detected (set option shell_type to override)", f.shellType) + f.m.Set("shell_type", f.shellType) } + // Ensure we have absolute path to root + // It appears that WS FTP doesn't like relative paths, + // and the openssh sftp tool also uses absolute paths. + if !path.IsAbs(f.root) { + path, err := c.sftpClient.RealPath(f.root) + if err != nil { + fs.Debugf(f, "Failed to resolve path - using relative paths: %v", err) + } else { + f.absRoot = path + fs.Debugf(f, "Relative path resolved to %q", f.absRoot) + } + } + f.putSftpConnection(&c, err) if root != "" { - // Check to see if the root actually an existing file + // Check to see if the root is actually an existing file, + // and if so change the filesystem root to its parent directory. oldAbsRoot := f.absRoot remote := path.Base(root) f.root = path.Dir(root) @@ -807,20 +878,24 @@ func NewFsWithConnection(ctx context.Context, f *Fs, name string, root string, m if f.root == "." { f.root = "" } - _, err := f.NewObject(ctx, remote) + _, err = f.NewObject(ctx, remote) if err != nil { - if err == fs.ErrorObjectNotFound || err == fs.ErrorIsDir { - // File doesn't exist so return old f - f.root = root - f.absRoot = oldAbsRoot - return f, nil + if err != fs.ErrorObjectNotFound && err != fs.ErrorIsDir { + return nil, err } - return nil, err + // File doesn't exist so keep the old f + f.root = root + f.absRoot = oldAbsRoot + err = nil + } else { + // File exists so change fs to point to the parent and return it with an error + err = fs.ErrorIsFile } - // return an error with an fs which points to the parent - return f, fs.ErrorIsFile + } else { + err = nil } - return f, nil + fs.Debugf(f, "Using root directory %q", f.absRoot) + return f, err } // Name returns the configured name of the file system @@ -1155,74 +1230,147 @@ func (f *Fs) run(ctx context.Context, cmd string) ([]byte, error) { // Hashes returns the supported hash types of the filesystem func (f *Fs) Hashes() hash.Set { ctx := context.TODO() - if f.opt.DisableHashCheck { - return hash.Set(hash.None) - } if f.cachedHashes != nil { return *f.cachedHashes } + hashSet := hash.NewHashSet() + f.cachedHashes = &hashSet + + if f.opt.DisableHashCheck || f.shellType == shellTypeNotSupported { + return hashSet + } + // look for a hash command which works - checkHash := func(commands []string, expected string, hashCommand *string, changed *bool) bool { + checkHash := func(hashType hash.Type, commands []struct{ hashFile, hashEmpty string }, expected string, hashCommand *string, changed *bool) bool { if *hashCommand == hashCommandNotSupported { return false } if *hashCommand != "" { return true } + fs.Debugf(f, "Checking default %v hash commands", hashType) *changed = true for _, command := range commands { - output, err := f.run(ctx, command) + output, err := f.run(ctx, command.hashEmpty) if err != nil { + fs.Debugf(f, "Hash command skipped: %v", err) continue } output = bytes.TrimSpace(output) - fs.Debugf(f, "checking %q command: %q", command, output) if parseHash(output) == expected { - *hashCommand = command + *hashCommand = command.hashFile + fs.Debugf(f, "Hash command accepted") return true } + fs.Debugf(f, "Hash command skipped: Wrong output") } *hashCommand = hashCommandNotSupported return false } changed := false - md5Works := checkHash([]string{"md5sum", "md5 -r", "rclone md5sum"}, "d41d8cd98f00b204e9800998ecf8427e", &f.opt.Md5sumCommand, &changed) - sha1Works := checkHash([]string{"sha1sum", "sha1 -r", "rclone sha1sum"}, "da39a3ee5e6b4b0d3255bfef95601890afd80709", &f.opt.Sha1sumCommand, &changed) + md5Commands := []struct { + hashFile, hashEmpty string + }{ + {"md5sum", "md5sum"}, + {"md5 -r", "md5 -r"}, + {"rclone md5sum", "rclone md5sum"}, + } + sha1Commands := []struct { + hashFile, hashEmpty string + }{ + {"sha1sum", "sha1sum"}, + {"sha1 -r", "sha1 -r"}, + {"rclone sha1sum", "rclone sha1sum"}, + } + if f.shellType == "powershell" { + md5Commands = append(md5Commands, struct { + hashFile, hashEmpty string + }{ + "&{param($Path);Get-FileHash -Algorithm MD5 -LiteralPath $Path -ErrorAction Stop|Select-Object -First 1 -ExpandProperty Hash|ForEach-Object{\"$($_.ToLower()) ${Path}\"}}", + "Get-FileHash -Algorithm MD5 -InputStream ([System.IO.MemoryStream]::new()) -ErrorAction Stop|Select-Object -First 1 -ExpandProperty Hash|ForEach-Object{$_.ToLower()}", + }) + + sha1Commands = append(sha1Commands, struct { + hashFile, hashEmpty string + }{ + "&{param($Path);Get-FileHash -Algorithm SHA1 -LiteralPath $Path -ErrorAction Stop|Select-Object -First 1 -ExpandProperty Hash|ForEach-Object{\"$($_.ToLower()) ${Path}\"}}", + "Get-FileHash -Algorithm SHA1 -InputStream ([System.IO.MemoryStream]::new()) -ErrorAction Stop|Select-Object -First 1 -ExpandProperty Hash|ForEach-Object{$_.ToLower()}", + }) + } + + md5Works := checkHash(hash.MD5, md5Commands, "d41d8cd98f00b204e9800998ecf8427e", &f.opt.Md5sumCommand, &changed) + sha1Works := checkHash(hash.SHA1, sha1Commands, "da39a3ee5e6b4b0d3255bfef95601890afd80709", &f.opt.Sha1sumCommand, &changed) if changed { + // Save permanently in config to avoid the extra work next time + fs.Debugf(f, "Setting hash command for %v to %q (set sha1sum_command to override)", hash.MD5, f.opt.Md5sumCommand) f.m.Set("md5sum_command", f.opt.Md5sumCommand) + fs.Debugf(f, "Setting hash command for %v to %q (set md5sum_command to override)", hash.SHA1, f.opt.Sha1sumCommand) f.m.Set("sha1sum_command", f.opt.Sha1sumCommand) } - set := hash.NewHashSet() if sha1Works { - set.Add(hash.SHA1) + hashSet.Add(hash.SHA1) } if md5Works { - set.Add(hash.MD5) + hashSet.Add(hash.MD5) } - f.cachedHashes = &set - return set + return hashSet } // About gets usage stats func (f *Fs) About(ctx context.Context) (*fs.Usage, error) { - escapedPath := shellEscape(f.root) - if f.opt.PathOverride != "" { - escapedPath = shellEscape(path.Join(f.opt.PathOverride, f.root)) + if f.shellType == shellTypeNotSupported || f.shellType == "cmd" { + fs.Debugf(f, "About shell command is not available for shell type %q (set option shell_type to override)", f.shellType) + return nil, fmt.Errorf("not supported with shell type %q", f.shellType) } - if len(escapedPath) == 0 { - escapedPath = "/" + aboutShellPath := f.remoteShellPath("") + if aboutShellPath == "" { + aboutShellPath = "/" } - stdout, err := f.run(ctx, "df -k "+escapedPath) + fs.Debugf(f, "About path %q", aboutShellPath) + aboutShellPathArg, err := f.quoteOrEscapeShellPath(aboutShellPath) if err != nil { + return nil, err + } + // PowerShell + if f.shellType == "powershell" { + shellCmd := "Get-Item " + aboutShellPathArg + " -ErrorAction Stop|Select-Object -First 1 -ExpandProperty PSDrive|ForEach-Object{\"$($_.Used) $($_.Free)\"}" + fs.Debugf(f, "About using shell command for shell type %q", f.shellType) + stdout, err := f.run(ctx, shellCmd) + if err != nil { + fs.Debugf(f, "About shell command for shell type %q failed (set option shell_type to override): %v", f.shellType, err) + return nil, fmt.Errorf("powershell command failed: %w", err) + } + split := strings.Fields(string(stdout)) + usage := &fs.Usage{} + if len(split) == 2 { + usedValue, usedErr := strconv.ParseInt(split[0], 10, 64) + if usedErr == nil { + usage.Used = fs.NewUsageValue(usedValue) + } + freeValue, freeErr := strconv.ParseInt(split[1], 10, 64) + if freeErr == nil { + usage.Free = fs.NewUsageValue(freeValue) + if usedErr == nil { + usage.Total = fs.NewUsageValue(usedValue + freeValue) + } + } + } + return usage, nil + } + // Unix/default shell + shellCmd := "df -k " + aboutShellPathArg + fs.Debugf(f, "About using shell command for shell type %q", f.shellType) + stdout, err := f.run(ctx, shellCmd) + if err != nil { + fs.Debugf(f, "About shell command for shell type %q failed (set option shell_type to override): %v", f.shellType, err) return nil, fmt.Errorf("your remote may not have the required df utility: %w", err) } - usageTotal, usageUsed, usageAvail := parseUsage(stdout) usage := &fs.Usage{} if usageTotal >= 0 { @@ -1287,31 +1435,78 @@ func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) { return "", hash.ErrUnsupported } - escapedPath := shellEscape(o.path()) - if o.fs.opt.PathOverride != "" { - escapedPath = shellEscape(path.Join(o.fs.opt.PathOverride, o.remote)) - } - b, err := o.fs.run(ctx, hashCmd+" "+escapedPath) + shellPathArg, err := o.fs.quoteOrEscapeShellPath(o.shellPath()) if err != nil { return "", fmt.Errorf("failed to calculate %v hash: %w", r, err) } - - str := parseHash(b) - if r == hash.MD5 { - o.md5sum = &str - } else if r == hash.SHA1 { - o.sha1sum = &str + outBytes, err := o.fs.run(ctx, hashCmd+" "+shellPathArg) + if err != nil { + return "", fmt.Errorf("failed to calculate %v hash: %w", r, err) } - return str, nil + hashString := parseHash(outBytes) + fs.Debugf(o, "Parsed hash: %s", hashString) + if r == hash.MD5 { + o.md5sum = &hashString + } else if r == hash.SHA1 { + o.sha1sum = &hashString + } + return hashString, nil } -var shellEscapeRegex = regexp.MustCompile("[^A-Za-z0-9_.,:/\\@\u0080-\uFFFFFFFF\n-]") +// quoteOrEscapeShellPath makes path a valid string argument in configured shell +// and also ensures it cannot cause unintended behavior. +func quoteOrEscapeShellPath(shellType string, shellPath string) (string, error) { + // PowerShell + if shellType == "powershell" { + return "'" + strings.ReplaceAll(shellPath, "'", "''") + "'", nil + } + // Windows Command Prompt + if shellType == "cmd" { + if strings.Contains(shellPath, "\"") { + return "", fmt.Errorf("path is not valid in shell type %s: %s", shellType, shellPath) + } + return "\"" + shellPath + "\"", nil + } + // Unix shell + safe := unixShellEscapeRegex.ReplaceAllString(shellPath, `\$0`) + return strings.ReplaceAll(safe, "\n", "'\n'"), nil +} -// Escape a string s.t. it cannot cause unintended behavior -// when sending it to a shell. -func shellEscape(str string) string { - safe := shellEscapeRegex.ReplaceAllString(str, `\$0`) - return strings.ReplaceAll(safe, "\n", "'\n'") +// quoteOrEscapeShellPath makes path a valid string argument in configured shell +func (f *Fs) quoteOrEscapeShellPath(shellPath string) (string, error) { + return quoteOrEscapeShellPath(f.shellType, shellPath) +} + +// remotePath returns the native SFTP path of the file or directory at the remote given +func (f *Fs) remotePath(remote string) string { + return path.Join(f.absRoot, remote) +} + +// remoteShellPath returns the SSH shell path of the file or directory at the remote given +func (f *Fs) remoteShellPath(remote string) string { + if f.opt.PathOverride != "" { + shellPath := path.Join(f.opt.PathOverride, remote) + fs.Debugf(f, "Shell path redirected to %q with option path_override", shellPath) + return shellPath + } + shellPath := path.Join(f.absRoot, remote) + if f.shellType == "powershell" || f.shellType == "cmd" { + // If remote shell is powershell or cmd, then server is probably Windows. + // The sftp package converts everything to POSIX paths: Forward slashes, and + // absolute paths starts with a slash. An absolute path on a Windows server will + // then look like this "/C:/Windows/System32". We must remove the "/" prefix + // to make this a valid path for shell commands. In case of PowerShell there is a + // possibility that it is a Unix server, with PowerShell Core shell, but assuming + // root folders with names such as "C:" are rare, we just take this risk, + // and option path_override can always be used to work around corner cases. + if posixWinAbsPathRegex.MatchString(shellPath) { + shellPath = strings.TrimPrefix(shellPath, "/") + fs.Debugf(f, "Shell path adjusted to %q (set option path_override to override)", shellPath) + return shellPath + } + } + fs.Debugf(f, "Shell path %q", shellPath) + return shellPath } // Converts a byte array from the SSH session returned by @@ -1362,9 +1557,14 @@ func (o *Object) ModTime(ctx context.Context) time.Time { return o.modTime } -// path returns the native path of the object +// path returns the native SFTP path of the object func (o *Object) path() string { - return path.Join(o.fs.absRoot, o.remote) + return o.fs.remotePath(o.remote) +} + +// shellPath returns the SSH shell path of the object +func (o *Object) shellPath() string { + return o.fs.remoteShellPath(o.remote) } // setMetadata updates the info in the object from the stat result passed in diff --git a/backend/sftp/sftp_internal_test.go b/backend/sftp/sftp_internal_test.go index e73cc417e..0803321c8 100644 --- a/backend/sftp/sftp_internal_test.go +++ b/backend/sftp/sftp_internal_test.go @@ -10,7 +10,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestShellEscape(t *testing.T) { +func TestShellEscapeUnix(t *testing.T) { for i, test := range []struct { unescaped, escaped string }{ @@ -20,7 +20,44 @@ func TestShellEscape(t *testing.T) { {"/test/\n", "/test/'\n'"}, {":\"'", ":\\\"\\'"}, } { - got := shellEscape(test.unescaped) + got, err := quoteOrEscapeShellPath("unix", test.unescaped) + assert.NoError(t, err) + assert.Equal(t, test.escaped, got, fmt.Sprintf("Test %d unescaped = %q", i, test.unescaped)) + } +} + +func TestShellEscapeCmd(t *testing.T) { + for i, test := range []struct { + unescaped, escaped string + ok bool + }{ + {"", "\"\"", true}, + {"c:/this/is/harmless", "\"c:/this/is/harmless\"", true}, + {"c:/test¬epad", "\"c:/test¬epad\"", true}, + {"c:/test\"&\"notepad", "", false}, + } { + got, err := quoteOrEscapeShellPath("cmd", test.unescaped) + if test.ok { + assert.NoError(t, err) + assert.Equal(t, test.escaped, got, fmt.Sprintf("Test %d unescaped = %q", i, test.unescaped)) + } else { + assert.Error(t, err) + } + } +} + +func TestShellEscapePowerShell(t *testing.T) { + for i, test := range []struct { + unescaped, escaped string + }{ + {"", "''"}, + {"c:/this/is/harmless", "'c:/this/is/harmless'"}, + {"c:/test¬epad", "'c:/test¬epad'"}, + {"c:/test\"&\"notepad", "'c:/test\"&\"notepad'"}, + {"c:/test'&'notepad", "'c:/test''&''notepad'"}, + } { + got, err := quoteOrEscapeShellPath("powershell", test.unescaped) + assert.NoError(t, err) assert.Equal(t, test.escaped, got, fmt.Sprintf("Test %d unescaped = %q", i, test.unescaped)) } } diff --git a/docs/content/sftp.md b/docs/content/sftp.md index 5bc27dcd8..ecd218990 100644 --- a/docs/content/sftp.md +++ b/docs/content/sftp.md @@ -28,6 +28,9 @@ Note that some SFTP servers will need the leading / - Synology is a good example of this. rsync.net, on the other hand, requires users to OMIT the leading /. +Note that by default rclone will try to execute shell commands on +the server, see [shell access considerations](#shell-access-considerations). + ## Configuration Here is an example of making an SFTP configuration. First run @@ -244,6 +247,116 @@ And then at the end of the session These commands can be used in scripts of course. +### Shell access + +Some functionality of the SFTP backend relies on remote shell access, +and the possibility to execute commands. This includes [checksum](#checksum), +and in some cases also [about](#about-command). The shell commands that +must be executed may be different on different type of shells, and also +quoting/escaping of file path arguments containing special characters may +be different. Rclone therefore needs to know what type of shell it is, +and if shell access is available at all. + +Most servers run on some version of Unix, and then a basic Unix shell can +be assumed, without further distinction. Windows 10, Server 2019, and later +can also run a SSH server, which is a port of OpenSSH (see official +[installation guide](https://docs.microsoft.com/en-us/windows-server/administration/openssh/openssh_install_firstuse)). On a Windows server the shell handling is different: Although it can also +be set up to use a Unix type shell, e.g. Cygwin bash, the default is to +use Windows Command Prompt (cmd.exe), and PowerShell is a recommended +alternative. All of these have bahave differently, which rclone must handle. + +Rclone tries to auto-detect what type of shell is used on the server, +first time you access the SFTP remote. If a remote shell session is +successfully created, it will look for indications that it is CMD or +PowerShell, with fall-back to Unix if not something else is detected. +If unable to even create a remote shell session, then shell command +execution will be disabled entirely. The result is stored in the SFTP +remote configuration, in option `shell_type`, so that the auto-detection +only have to be performed once. If you manually set a value for this +option before first run, the auto-detection will be skipped, and if +you set a different value later this will override any existing. +Value `none` can be set to avoid any attempts at executing shell +commands, e.g. if this is not allowed on the server. + +When the server is [rclone serve sftp](/commands/rclone_serve_sftp/), +the rclone SFTP remote will detect this as a Unix type shell - even +if it is running on Windows. This server does not actually have a shell, +but it accepts input commands matching the specific ones that the +SFTP backend relies on for Unix shells, e.g. `md5sum` and `df`. Also +it handles the string escape rules used for Unix shell. Treating it +as a Unix type shell from a SFTP remote will therefore always be +correct, and support all features. + +#### Shell access considerations + +The shell type auto-detection logic, described above, means that +by default rclone will try to run a shell command the first time +a new sftp remote is accessed. If you configure a sftp remote +without a config file, e.g. an [on the fly](/docs/#backend-path-to-dir]) +remote, rclone will have nowhere to store the result, and it +will re-run the command on every access. To avoid this you should +explicitely set the `shell_type` option to the correct value, +or to `none` if you want to prevent rclone from executing any +remote shell commands. + +It is also important to note that, since the shell type decides +how quoting and escaping of file paths used as command-line arguments +are performed, configuring the wrong shell type may leave you exposed +to command injection exploits. Make sure to confirm the auto-detected +shell type, or explicitely set the shell type you know is correct, +or disable shell access until you know. + +### Checksum + +SFTP does not natively support checksums (file hash), but rclone +is able to use checksumming if the same login has shell access, +and can execute remote commands. If there is a command that can +calculate compatible checksums on the remote system, Rclone can +then be configured to execute this whenever a checksum is needed, +and read back the results. Currently MD5 and SHA-1 are supported. + +Normally this requires an external utility being available on +the server. By default rclone will try commands `md5sum`, `md5` +and `rclone md5sum` for MD5 checksums, and the first one found usable +will be picked. Same with `sha1sum`, `sha1` and `rclone sha1sum` +commands for SHA-1 checksums. These utilities normally need to +be in the remote's PATH to be found. + +In some cases the shell itself is capable of calculating checksums. +PowerShell is an example of such a shell. If rclone detects that the +remote shell is PowerShell, which means it most probably is a +Windows OpenSSH server, rclone will use a predefined script block +to produce the checksums when no external checksum commands are found +(see [shell access](#shell-access)). This assumes PowerShell version +4.0 or newer. + +The options `md5sum_command` and `sha1_command` can be used to customize +the command to be executed for calculation of checksums. You can for +example set a specific path to where md5sum and sha1sum executables +are located, or use them to specify some other tools that print checksums +in compatible format. The value can include command-line arguments, +or even shell script blocks as with PowerShell. Rclone has subcommands +[md5sum](/commands/rclone_md5sum/) and [sha1sum](/commands/rclone_sha1sum/) +that use compatible format, which means if you have an rclone executable +on the server it can be used. As mentioned above, they will be automatically +picked up if found in PATH, but if not you can set something like +`/path/to/rclone md5sum` as the value of option `md5sum_command` to +make sure a specific executable is used. + +Remote checksumming is recommended and enabled by default. First time +rclone is using a SFTP remote, if options `md5sum_command` or `sha1_command` +are not set, it will check if any of the default commands for each of them, +as described above, can be used. The result will be saved in the remote +configuration, so next time it will use the same. Value `none` +will be set if none of the default commands could be used for a specific +algorithm, and this algorithm will not be supported by the remote. + +Disabling the checksumming may be required if you are connecting to SFTP servers +which are not under your control, and to which the execution of remote shell +commands is prohibited. Set the configuration option `disable_hashcheck` +to `true` to disable checksumming entirely, or set `shell_type` to `none` +to disable all functionality based on remote shell command execution. + ### Modified time Modified times are stored on the server to 1 second precision. @@ -255,6 +368,20 @@ upload (for example, certain configurations of ProFTPd with mod_sftp). If you are using one of these servers, you can set the option `set_modtime = false` in your RClone backend configuration to disable this behaviour. +### About command + +SFTP supports the [about](/commands/rclone_about/) command if the +same login has access to a Unix shell, where the `df` command is +available (e.g. in the remote's PATH). `about` will return the +total space, free space, and used space on the remote for the disk +of the specified path on the remote or, if not set, the disk of +the root on the remote. `about` will fail if it does not have +shell access or if `df` is not found. + +If the server shell is PowerShell, probably with a Windows OpenSSH +server, rclone supports `about` using a built-in shell command +(see [shell access](#shell-access)). + {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/sftp/sftp.go then run make backenddocs" >}} ### Standard options @@ -637,25 +764,9 @@ Properties: ## Limitations -SFTP supports checksums if the same login has shell access and `md5sum` -or `sha1sum` as well as `echo` are in the remote's PATH. -This remote checksumming (file hashing) is recommended and enabled by default. -Disabling the checksumming may be required if you are connecting to SFTP servers -which are not under your control, and to which the execution of remote commands -is prohibited. Set the configuration option `disable_hashcheck` to `true` to -disable checksumming. - -SFTP also supports `about` if the same login has shell -access and `df` are in the remote's PATH. `about` will -return the total space, free space, and used space on the remote -for the disk of the specified path on the remote or, if not set, -the disk of the root on the remote. -`about` will fail if it does not have shell -access or if `df` is not in the remote's PATH. - -Note that some SFTP servers (e.g. Synology) the paths are different for -SSH and SFTP so the hashes can't be calculated properly. For them -using `disable_hashcheck` is a good idea. +On some SFTP servers (e.g. Synology) the paths are different +for SSH and SFTP so the hashes can't be calculated properly. +For them using `disable_hashcheck` is a good idea. The only ssh agent supported under Windows is Putty's pageant. @@ -670,11 +781,10 @@ SFTP isn't supported under plan9 until [this issue](https://github.com/pkg/sftp/issues/156) is fixed. Note that since SFTP isn't HTTP based the following flags don't work -with it: `--dump-headers`, `--dump-bodies`, `--dump-auth` +with it: `--dump-headers`, `--dump-bodies`, `--dump-auth`. Note that `--timeout` and `--contimeout` are both supported. - ## rsync.net {#rsync-net} rsync.net is supported through the SFTP backend.