sftp: add support for about and hashsum on windows server
Windows shells like cmd and powershell needs to use different quoting/escaping of strings and paths than the unix shell, and also absolute paths must be fixed by removing leading slash that the POSIX formatted paths have (e.g. /C:/Users does not work in shell, it must be converted to C:/Users). Tries to autodetect shell type (cmd, powershell, unix) on first use. Implemented default builtin powershell functions for hashsum and about when remote shell is powershell. See #5763 Fixes #5758
This commit is contained in:
albertony
parent
218bf2183d
commit
b4091f282a
|
|
@ -39,6 +39,8 @@ import (
|
|||
)
|
||||
|
||||
const (
|
||||
defaultShellType = "unix"
|
||||
shellTypeNotSupported = "none"
|
||||
hashCommandNotSupported = "none"
|
||||
minSleep = 100 * time.Millisecond
|
||||
maxSleep = 2 * time.Second
|
||||
|
|
@ -47,7 +49,9 @@ const (
|
|||
)
|
||||
|
||||
var (
|
||||
currentUser = env.CurrentUser()
|
||||
currentUser = env.CurrentUser()
|
||||
posixWinAbsPathRegex = regexp.MustCompile(`^/[a-zA-Z]\:($|/)`) // E.g. "/C:" or anything starting with "/C:/"
|
||||
unixShellEscapeRegex = regexp.MustCompile("[^A-Za-z0-9_.,:/\\@\u0080-\uFFFFFFFF\n-]")
|
||||
)
|
||||
|
||||
func init() {
|
||||
|
|
@ -148,16 +152,16 @@ If this is set and no password is supplied then rclone will:
|
|||
}, {
|
||||
Name: "path_override",
|
||||
Default: "",
|
||||
Help: `Override path used by SSH connection.
|
||||
Help: `Override path used by SSH shell commands.
|
||||
|
||||
This allows checksum calculation when SFTP and SSH paths are
|
||||
different. This issue affects among others Synology NAS boxes.
|
||||
|
||||
Shared folders can be found in directories representing volumes
|
||||
E.g. if shared folders can be found in directories representing volumes:
|
||||
|
||||
rclone sync /home/local/directory remote:/directory --sftp-path-override /volume2/directory
|
||||
|
||||
Home directory can be found in a shared folder called "home"
|
||||
E.g. if home directory can be found in a shared folder called "home":
|
||||
|
||||
rclone sync /home/local/directory remote:/home/directory --sftp-path-override /volume1/homes/USER/directory`,
|
||||
Advanced: true,
|
||||
|
|
@ -166,6 +170,26 @@ Home directory can be found in a shared folder called "home"
|
|||
Default: true,
|
||||
Help: "Set the modified time on the remote if set.",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "shell_type",
|
||||
Default: "",
|
||||
Help: "The type of SSH shell on remote server, if any.\n\nLeave blank for autodetect.",
|
||||
Advanced: true,
|
||||
Examples: []fs.OptionExample{
|
||||
{
|
||||
Value: shellTypeNotSupported,
|
||||
Help: "No shell access",
|
||||
}, {
|
||||
Value: "unix",
|
||||
Help: "Unix shell",
|
||||
}, {
|
||||
Value: "powershell",
|
||||
Help: "PowerShell",
|
||||
}, {
|
||||
Value: "cmd",
|
||||
Help: "Windows Command Prompt",
|
||||
},
|
||||
},
|
||||
}, {
|
||||
Name: "md5sum_command",
|
||||
Default: "",
|
||||
|
|
@ -270,6 +294,7 @@ type Options struct {
|
|||
AskPassword bool `config:"ask_password"`
|
||||
PathOverride string `config:"path_override"`
|
||||
SetModTime bool `config:"set_modtime"`
|
||||
ShellType string `config:"shell_type"`
|
||||
Md5sumCommand string `config:"md5sum_command"`
|
||||
Sha1sumCommand string `config:"sha1sum_command"`
|
||||
SkipLinks bool `config:"skip_links"`
|
||||
|
|
@ -286,6 +311,8 @@ type Fs struct {
|
|||
name string
|
||||
root string
|
||||
absRoot string
|
||||
shellRoot string
|
||||
shellType string
|
||||
opt Options // parsed options
|
||||
ci *fs.ConfigInfo // global config
|
||||
m configmap.Mapper // config
|
||||
|
|
@ -542,7 +569,7 @@ func (f *Fs) drainPool(ctx context.Context) (err error) {
|
|||
f.drain.Stop()
|
||||
}
|
||||
if len(f.pool) != 0 {
|
||||
fs.Debugf(f, "closing %d unused connections", len(f.pool))
|
||||
fs.Debugf(f, "Closing %d unused connections", len(f.pool))
|
||||
}
|
||||
for i, c := range f.pool {
|
||||
if cErr := c.closed(); cErr == nil {
|
||||
|
|
@ -739,7 +766,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
|||
//
|
||||
// Just send the password back for all questions
|
||||
func (f *Fs) keyboardInteractiveReponse(user, instruction string, questions []string, echos []bool, pass string) ([]string, error) {
|
||||
fs.Debugf(f, "keyboard interactive auth requested")
|
||||
fs.Debugf(f, "Keyboard interactive auth requested")
|
||||
answers := make([]string, len(questions))
|
||||
for i := range answers {
|
||||
answers[i] = pass
|
||||
|
|
@ -769,6 +796,7 @@ func NewFsWithConnection(ctx context.Context, f *Fs, name string, root string, m
|
|||
f.name = name
|
||||
f.root = root
|
||||
f.absRoot = root
|
||||
f.shellRoot = root
|
||||
f.opt = *opt
|
||||
f.m = m
|
||||
f.config = sshConfig
|
||||
|
|
@ -778,7 +806,7 @@ func NewFsWithConnection(ctx context.Context, f *Fs, name string, root string, m
|
|||
f.savedpswd = ""
|
||||
// set the pool drainer timer going
|
||||
if f.opt.IdleTimeout > 0 {
|
||||
f.drain = time.AfterFunc(time.Duration(opt.IdleTimeout), func() { _ = f.drainPool(ctx) })
|
||||
f.drain = time.AfterFunc(time.Duration(f.opt.IdleTimeout), func() { _ = f.drainPool(ctx) })
|
||||
}
|
||||
|
||||
f.features = (&fs.Features{
|
||||
|
|
@ -790,16 +818,59 @@ func NewFsWithConnection(ctx context.Context, f *Fs, name string, root string, m
|
|||
if err != nil {
|
||||
return nil, fmt.Errorf("NewFs: %w", err)
|
||||
}
|
||||
cwd, err := c.sftpClient.Getwd()
|
||||
f.putSftpConnection(&c, nil)
|
||||
if err != nil {
|
||||
fs.Debugf(f, "Failed to read current directory - using relative paths: %v", err)
|
||||
} else if !path.IsAbs(f.root) {
|
||||
f.absRoot = path.Join(cwd, f.root)
|
||||
fs.Debugf(f, "Using absolute root directory %q", f.absRoot)
|
||||
// Check remote shell type, try to auto-detect if not configured and save to config for later
|
||||
if f.opt.ShellType != "" {
|
||||
f.shellType = f.opt.ShellType
|
||||
fs.Debugf(f, "Shell type %q from config", f.shellType)
|
||||
} else {
|
||||
session, err := c.sshClient.NewSession()
|
||||
if err != nil {
|
||||
f.shellType = shellTypeNotSupported
|
||||
fs.Debugf(f, "Failed to get shell session for shell type detection command: %v", err)
|
||||
} else {
|
||||
var stdout, stderr bytes.Buffer
|
||||
session.Stdout = &stdout
|
||||
session.Stderr = &stderr
|
||||
shellCmd := "echo ${ShellId}%ComSpec%"
|
||||
fs.Debugf(f, "Running shell type detection remote command: %s", shellCmd)
|
||||
err = session.Run(shellCmd)
|
||||
_ = session.Close()
|
||||
if err != nil {
|
||||
f.shellType = defaultShellType
|
||||
fs.Debugf(f, "Remote command failed: %v (stdout=%v) (stderr=%v)", err, bytes.TrimSpace(stdout.Bytes()), bytes.TrimSpace(stderr.Bytes()))
|
||||
} else {
|
||||
outBytes := stdout.Bytes()
|
||||
fs.Debugf(f, "Remote command result: %s", outBytes)
|
||||
outString := string(bytes.TrimSpace(stdout.Bytes()))
|
||||
if strings.HasPrefix(outString, "Microsoft.PowerShell") { // If PowerShell: "Microsoft.PowerShell%ComSpec%"
|
||||
f.shellType = "powershell"
|
||||
} else if !strings.HasSuffix(outString, "%ComSpec%") { // If Command Prompt: "${ShellId}C:\WINDOWS\system32\cmd.exe"
|
||||
f.shellType = "cmd"
|
||||
} else { // If Unix: "%ComSpec%"
|
||||
f.shellType = "unix"
|
||||
}
|
||||
}
|
||||
}
|
||||
// Save permanently in config to avoid the extra work next time
|
||||
fs.Debugf(f, "Shell type %q detected (set option shell_type to override)", f.shellType)
|
||||
f.m.Set("shell_type", f.shellType)
|
||||
}
|
||||
// Ensure we have absolute path to root
|
||||
// It appears that WS FTP doesn't like relative paths,
|
||||
// and the openssh sftp tool also uses absolute paths.
|
||||
if !path.IsAbs(f.root) {
|
||||
path, err := c.sftpClient.RealPath(f.root)
|
||||
if err != nil {
|
||||
fs.Debugf(f, "Failed to resolve path - using relative paths: %v", err)
|
||||
} else {
|
||||
f.absRoot = path
|
||||
fs.Debugf(f, "Relative path resolved to %q", f.absRoot)
|
||||
}
|
||||
}
|
||||
f.putSftpConnection(&c, err)
|
||||
if root != "" {
|
||||
// Check to see if the root actually an existing file
|
||||
// Check to see if the root is actually an existing file,
|
||||
// and if so change the filesystem root to its parent directory.
|
||||
oldAbsRoot := f.absRoot
|
||||
remote := path.Base(root)
|
||||
f.root = path.Dir(root)
|
||||
|
|
@ -807,20 +878,24 @@ func NewFsWithConnection(ctx context.Context, f *Fs, name string, root string, m
|
|||
if f.root == "." {
|
||||
f.root = ""
|
||||
}
|
||||
_, err := f.NewObject(ctx, remote)
|
||||
_, err = f.NewObject(ctx, remote)
|
||||
if err != nil {
|
||||
if err == fs.ErrorObjectNotFound || err == fs.ErrorIsDir {
|
||||
// File doesn't exist so return old f
|
||||
f.root = root
|
||||
f.absRoot = oldAbsRoot
|
||||
return f, nil
|
||||
if err != fs.ErrorObjectNotFound && err != fs.ErrorIsDir {
|
||||
return nil, err
|
||||
}
|
||||
return nil, err
|
||||
// File doesn't exist so keep the old f
|
||||
f.root = root
|
||||
f.absRoot = oldAbsRoot
|
||||
err = nil
|
||||
} else {
|
||||
// File exists so change fs to point to the parent and return it with an error
|
||||
err = fs.ErrorIsFile
|
||||
}
|
||||
// return an error with an fs which points to the parent
|
||||
return f, fs.ErrorIsFile
|
||||
} else {
|
||||
err = nil
|
||||
}
|
||||
return f, nil
|
||||
fs.Debugf(f, "Using root directory %q", f.absRoot)
|
||||
return f, err
|
||||
}
|
||||
|
||||
// Name returns the configured name of the file system
|
||||
|
|
@ -1155,74 +1230,147 @@ func (f *Fs) run(ctx context.Context, cmd string) ([]byte, error) {
|
|||
// Hashes returns the supported hash types of the filesystem
|
||||
func (f *Fs) Hashes() hash.Set {
|
||||
ctx := context.TODO()
|
||||
if f.opt.DisableHashCheck {
|
||||
return hash.Set(hash.None)
|
||||
}
|
||||
|
||||
if f.cachedHashes != nil {
|
||||
return *f.cachedHashes
|
||||
}
|
||||
|
||||
hashSet := hash.NewHashSet()
|
||||
f.cachedHashes = &hashSet
|
||||
|
||||
if f.opt.DisableHashCheck || f.shellType == shellTypeNotSupported {
|
||||
return hashSet
|
||||
}
|
||||
|
||||
// look for a hash command which works
|
||||
checkHash := func(commands []string, expected string, hashCommand *string, changed *bool) bool {
|
||||
checkHash := func(hashType hash.Type, commands []struct{ hashFile, hashEmpty string }, expected string, hashCommand *string, changed *bool) bool {
|
||||
if *hashCommand == hashCommandNotSupported {
|
||||
return false
|
||||
}
|
||||
if *hashCommand != "" {
|
||||
return true
|
||||
}
|
||||
fs.Debugf(f, "Checking default %v hash commands", hashType)
|
||||
*changed = true
|
||||
for _, command := range commands {
|
||||
output, err := f.run(ctx, command)
|
||||
output, err := f.run(ctx, command.hashEmpty)
|
||||
if err != nil {
|
||||
fs.Debugf(f, "Hash command skipped: %v", err)
|
||||
continue
|
||||
}
|
||||
output = bytes.TrimSpace(output)
|
||||
fs.Debugf(f, "checking %q command: %q", command, output)
|
||||
if parseHash(output) == expected {
|
||||
*hashCommand = command
|
||||
*hashCommand = command.hashFile
|
||||
fs.Debugf(f, "Hash command accepted")
|
||||
return true
|
||||
}
|
||||
fs.Debugf(f, "Hash command skipped: Wrong output")
|
||||
}
|
||||
*hashCommand = hashCommandNotSupported
|
||||
return false
|
||||
}
|
||||
|
||||
changed := false
|
||||
md5Works := checkHash([]string{"md5sum", "md5 -r", "rclone md5sum"}, "d41d8cd98f00b204e9800998ecf8427e", &f.opt.Md5sumCommand, &changed)
|
||||
sha1Works := checkHash([]string{"sha1sum", "sha1 -r", "rclone sha1sum"}, "da39a3ee5e6b4b0d3255bfef95601890afd80709", &f.opt.Sha1sumCommand, &changed)
|
||||
md5Commands := []struct {
|
||||
hashFile, hashEmpty string
|
||||
}{
|
||||
{"md5sum", "md5sum"},
|
||||
{"md5 -r", "md5 -r"},
|
||||
{"rclone md5sum", "rclone md5sum"},
|
||||
}
|
||||
sha1Commands := []struct {
|
||||
hashFile, hashEmpty string
|
||||
}{
|
||||
{"sha1sum", "sha1sum"},
|
||||
{"sha1 -r", "sha1 -r"},
|
||||
{"rclone sha1sum", "rclone sha1sum"},
|
||||
}
|
||||
if f.shellType == "powershell" {
|
||||
md5Commands = append(md5Commands, struct {
|
||||
hashFile, hashEmpty string
|
||||
}{
|
||||
"&{param($Path);Get-FileHash -Algorithm MD5 -LiteralPath $Path -ErrorAction Stop|Select-Object -First 1 -ExpandProperty Hash|ForEach-Object{\"$($_.ToLower()) ${Path}\"}}",
|
||||
"Get-FileHash -Algorithm MD5 -InputStream ([System.IO.MemoryStream]::new()) -ErrorAction Stop|Select-Object -First 1 -ExpandProperty Hash|ForEach-Object{$_.ToLower()}",
|
||||
})
|
||||
|
||||
sha1Commands = append(sha1Commands, struct {
|
||||
hashFile, hashEmpty string
|
||||
}{
|
||||
"&{param($Path);Get-FileHash -Algorithm SHA1 -LiteralPath $Path -ErrorAction Stop|Select-Object -First 1 -ExpandProperty Hash|ForEach-Object{\"$($_.ToLower()) ${Path}\"}}",
|
||||
"Get-FileHash -Algorithm SHA1 -InputStream ([System.IO.MemoryStream]::new()) -ErrorAction Stop|Select-Object -First 1 -ExpandProperty Hash|ForEach-Object{$_.ToLower()}",
|
||||
})
|
||||
}
|
||||
|
||||
md5Works := checkHash(hash.MD5, md5Commands, "d41d8cd98f00b204e9800998ecf8427e", &f.opt.Md5sumCommand, &changed)
|
||||
sha1Works := checkHash(hash.SHA1, sha1Commands, "da39a3ee5e6b4b0d3255bfef95601890afd80709", &f.opt.Sha1sumCommand, &changed)
|
||||
|
||||
if changed {
|
||||
// Save permanently in config to avoid the extra work next time
|
||||
fs.Debugf(f, "Setting hash command for %v to %q (set sha1sum_command to override)", hash.MD5, f.opt.Md5sumCommand)
|
||||
f.m.Set("md5sum_command", f.opt.Md5sumCommand)
|
||||
fs.Debugf(f, "Setting hash command for %v to %q (set md5sum_command to override)", hash.SHA1, f.opt.Sha1sumCommand)
|
||||
f.m.Set("sha1sum_command", f.opt.Sha1sumCommand)
|
||||
}
|
||||
|
||||
set := hash.NewHashSet()
|
||||
if sha1Works {
|
||||
set.Add(hash.SHA1)
|
||||
hashSet.Add(hash.SHA1)
|
||||
}
|
||||
if md5Works {
|
||||
set.Add(hash.MD5)
|
||||
hashSet.Add(hash.MD5)
|
||||
}
|
||||
|
||||
f.cachedHashes = &set
|
||||
return set
|
||||
return hashSet
|
||||
}
|
||||
|
||||
// About gets usage stats
|
||||
func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
|
||||
escapedPath := shellEscape(f.root)
|
||||
if f.opt.PathOverride != "" {
|
||||
escapedPath = shellEscape(path.Join(f.opt.PathOverride, f.root))
|
||||
if f.shellType == shellTypeNotSupported || f.shellType == "cmd" {
|
||||
fs.Debugf(f, "About shell command is not available for shell type %q (set option shell_type to override)", f.shellType)
|
||||
return nil, fmt.Errorf("not supported with shell type %q", f.shellType)
|
||||
}
|
||||
if len(escapedPath) == 0 {
|
||||
escapedPath = "/"
|
||||
aboutShellPath := f.remoteShellPath("")
|
||||
if aboutShellPath == "" {
|
||||
aboutShellPath = "/"
|
||||
}
|
||||
stdout, err := f.run(ctx, "df -k "+escapedPath)
|
||||
fs.Debugf(f, "About path %q", aboutShellPath)
|
||||
aboutShellPathArg, err := f.quoteOrEscapeShellPath(aboutShellPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// PowerShell
|
||||
if f.shellType == "powershell" {
|
||||
shellCmd := "Get-Item " + aboutShellPathArg + " -ErrorAction Stop|Select-Object -First 1 -ExpandProperty PSDrive|ForEach-Object{\"$($_.Used) $($_.Free)\"}"
|
||||
fs.Debugf(f, "About using shell command for shell type %q", f.shellType)
|
||||
stdout, err := f.run(ctx, shellCmd)
|
||||
if err != nil {
|
||||
fs.Debugf(f, "About shell command for shell type %q failed (set option shell_type to override): %v", f.shellType, err)
|
||||
return nil, fmt.Errorf("powershell command failed: %w", err)
|
||||
}
|
||||
split := strings.Fields(string(stdout))
|
||||
usage := &fs.Usage{}
|
||||
if len(split) == 2 {
|
||||
usedValue, usedErr := strconv.ParseInt(split[0], 10, 64)
|
||||
if usedErr == nil {
|
||||
usage.Used = fs.NewUsageValue(usedValue)
|
||||
}
|
||||
freeValue, freeErr := strconv.ParseInt(split[1], 10, 64)
|
||||
if freeErr == nil {
|
||||
usage.Free = fs.NewUsageValue(freeValue)
|
||||
if usedErr == nil {
|
||||
usage.Total = fs.NewUsageValue(usedValue + freeValue)
|
||||
}
|
||||
}
|
||||
}
|
||||
return usage, nil
|
||||
}
|
||||
// Unix/default shell
|
||||
shellCmd := "df -k " + aboutShellPathArg
|
||||
fs.Debugf(f, "About using shell command for shell type %q", f.shellType)
|
||||
stdout, err := f.run(ctx, shellCmd)
|
||||
if err != nil {
|
||||
fs.Debugf(f, "About shell command for shell type %q failed (set option shell_type to override): %v", f.shellType, err)
|
||||
return nil, fmt.Errorf("your remote may not have the required df utility: %w", err)
|
||||
}
|
||||
|
||||
usageTotal, usageUsed, usageAvail := parseUsage(stdout)
|
||||
usage := &fs.Usage{}
|
||||
if usageTotal >= 0 {
|
||||
|
|
@ -1287,31 +1435,78 @@ func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
|
|||
return "", hash.ErrUnsupported
|
||||
}
|
||||
|
||||
escapedPath := shellEscape(o.path())
|
||||
if o.fs.opt.PathOverride != "" {
|
||||
escapedPath = shellEscape(path.Join(o.fs.opt.PathOverride, o.remote))
|
||||
}
|
||||
b, err := o.fs.run(ctx, hashCmd+" "+escapedPath)
|
||||
shellPathArg, err := o.fs.quoteOrEscapeShellPath(o.shellPath())
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to calculate %v hash: %w", r, err)
|
||||
}
|
||||
|
||||
str := parseHash(b)
|
||||
if r == hash.MD5 {
|
||||
o.md5sum = &str
|
||||
} else if r == hash.SHA1 {
|
||||
o.sha1sum = &str
|
||||
outBytes, err := o.fs.run(ctx, hashCmd+" "+shellPathArg)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to calculate %v hash: %w", r, err)
|
||||
}
|
||||
return str, nil
|
||||
hashString := parseHash(outBytes)
|
||||
fs.Debugf(o, "Parsed hash: %s", hashString)
|
||||
if r == hash.MD5 {
|
||||
o.md5sum = &hashString
|
||||
} else if r == hash.SHA1 {
|
||||
o.sha1sum = &hashString
|
||||
}
|
||||
return hashString, nil
|
||||
}
|
||||
|
||||
var shellEscapeRegex = regexp.MustCompile("[^A-Za-z0-9_.,:/\\@\u0080-\uFFFFFFFF\n-]")
|
||||
// quoteOrEscapeShellPath makes path a valid string argument in configured shell
|
||||
// and also ensures it cannot cause unintended behavior.
|
||||
func quoteOrEscapeShellPath(shellType string, shellPath string) (string, error) {
|
||||
// PowerShell
|
||||
if shellType == "powershell" {
|
||||
return "'" + strings.ReplaceAll(shellPath, "'", "''") + "'", nil
|
||||
}
|
||||
// Windows Command Prompt
|
||||
if shellType == "cmd" {
|
||||
if strings.Contains(shellPath, "\"") {
|
||||
return "", fmt.Errorf("path is not valid in shell type %s: %s", shellType, shellPath)
|
||||
}
|
||||
return "\"" + shellPath + "\"", nil
|
||||
}
|
||||
// Unix shell
|
||||
safe := unixShellEscapeRegex.ReplaceAllString(shellPath, `\$0`)
|
||||
return strings.ReplaceAll(safe, "\n", "'\n'"), nil
|
||||
}
|
||||
|
||||
// Escape a string s.t. it cannot cause unintended behavior
|
||||
// when sending it to a shell.
|
||||
func shellEscape(str string) string {
|
||||
safe := shellEscapeRegex.ReplaceAllString(str, `\$0`)
|
||||
return strings.ReplaceAll(safe, "\n", "'\n'")
|
||||
// quoteOrEscapeShellPath makes path a valid string argument in configured shell
|
||||
func (f *Fs) quoteOrEscapeShellPath(shellPath string) (string, error) {
|
||||
return quoteOrEscapeShellPath(f.shellType, shellPath)
|
||||
}
|
||||
|
||||
// remotePath returns the native SFTP path of the file or directory at the remote given
|
||||
func (f *Fs) remotePath(remote string) string {
|
||||
return path.Join(f.absRoot, remote)
|
||||
}
|
||||
|
||||
// remoteShellPath returns the SSH shell path of the file or directory at the remote given
|
||||
func (f *Fs) remoteShellPath(remote string) string {
|
||||
if f.opt.PathOverride != "" {
|
||||
shellPath := path.Join(f.opt.PathOverride, remote)
|
||||
fs.Debugf(f, "Shell path redirected to %q with option path_override", shellPath)
|
||||
return shellPath
|
||||
}
|
||||
shellPath := path.Join(f.absRoot, remote)
|
||||
if f.shellType == "powershell" || f.shellType == "cmd" {
|
||||
// If remote shell is powershell or cmd, then server is probably Windows.
|
||||
// The sftp package converts everything to POSIX paths: Forward slashes, and
|
||||
// absolute paths starts with a slash. An absolute path on a Windows server will
|
||||
// then look like this "/C:/Windows/System32". We must remove the "/" prefix
|
||||
// to make this a valid path for shell commands. In case of PowerShell there is a
|
||||
// possibility that it is a Unix server, with PowerShell Core shell, but assuming
|
||||
// root folders with names such as "C:" are rare, we just take this risk,
|
||||
// and option path_override can always be used to work around corner cases.
|
||||
if posixWinAbsPathRegex.MatchString(shellPath) {
|
||||
shellPath = strings.TrimPrefix(shellPath, "/")
|
||||
fs.Debugf(f, "Shell path adjusted to %q (set option path_override to override)", shellPath)
|
||||
return shellPath
|
||||
}
|
||||
}
|
||||
fs.Debugf(f, "Shell path %q", shellPath)
|
||||
return shellPath
|
||||
}
|
||||
|
||||
// Converts a byte array from the SSH session returned by
|
||||
|
|
@ -1362,9 +1557,14 @@ func (o *Object) ModTime(ctx context.Context) time.Time {
|
|||
return o.modTime
|
||||
}
|
||||
|
||||
// path returns the native path of the object
|
||||
// path returns the native SFTP path of the object
|
||||
func (o *Object) path() string {
|
||||
return path.Join(o.fs.absRoot, o.remote)
|
||||
return o.fs.remotePath(o.remote)
|
||||
}
|
||||
|
||||
// shellPath returns the SSH shell path of the object
|
||||
func (o *Object) shellPath() string {
|
||||
return o.fs.remoteShellPath(o.remote)
|
||||
}
|
||||
|
||||
// setMetadata updates the info in the object from the stat result passed in
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import (
|
|||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestShellEscape(t *testing.T) {
|
||||
func TestShellEscapeUnix(t *testing.T) {
|
||||
for i, test := range []struct {
|
||||
unescaped, escaped string
|
||||
}{
|
||||
|
|
@ -20,7 +20,44 @@ func TestShellEscape(t *testing.T) {
|
|||
{"/test/\n", "/test/'\n'"},
|
||||
{":\"'", ":\\\"\\'"},
|
||||
} {
|
||||
got := shellEscape(test.unescaped)
|
||||
got, err := quoteOrEscapeShellPath("unix", test.unescaped)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, test.escaped, got, fmt.Sprintf("Test %d unescaped = %q", i, test.unescaped))
|
||||
}
|
||||
}
|
||||
|
||||
func TestShellEscapeCmd(t *testing.T) {
|
||||
for i, test := range []struct {
|
||||
unescaped, escaped string
|
||||
ok bool
|
||||
}{
|
||||
{"", "\"\"", true},
|
||||
{"c:/this/is/harmless", "\"c:/this/is/harmless\"", true},
|
||||
{"c:/test¬epad", "\"c:/test¬epad\"", true},
|
||||
{"c:/test\"&\"notepad", "", false},
|
||||
} {
|
||||
got, err := quoteOrEscapeShellPath("cmd", test.unescaped)
|
||||
if test.ok {
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, test.escaped, got, fmt.Sprintf("Test %d unescaped = %q", i, test.unescaped))
|
||||
} else {
|
||||
assert.Error(t, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestShellEscapePowerShell(t *testing.T) {
|
||||
for i, test := range []struct {
|
||||
unescaped, escaped string
|
||||
}{
|
||||
{"", "''"},
|
||||
{"c:/this/is/harmless", "'c:/this/is/harmless'"},
|
||||
{"c:/test¬epad", "'c:/test¬epad'"},
|
||||
{"c:/test\"&\"notepad", "'c:/test\"&\"notepad'"},
|
||||
{"c:/test'&'notepad", "'c:/test''&''notepad'"},
|
||||
} {
|
||||
got, err := quoteOrEscapeShellPath("powershell", test.unescaped)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, test.escaped, got, fmt.Sprintf("Test %d unescaped = %q", i, test.unescaped))
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -28,6 +28,9 @@ Note that some SFTP servers will need the leading / - Synology is a
|
|||
good example of this. rsync.net, on the other hand, requires users to
|
||||
OMIT the leading /.
|
||||
|
||||
Note that by default rclone will try to execute shell commands on
|
||||
the server, see [shell access considerations](#shell-access-considerations).
|
||||
|
||||
## Configuration
|
||||
|
||||
Here is an example of making an SFTP configuration. First run
|
||||
|
|
@ -244,6 +247,116 @@ And then at the end of the session
|
|||
|
||||
These commands can be used in scripts of course.
|
||||
|
||||
### Shell access
|
||||
|
||||
Some functionality of the SFTP backend relies on remote shell access,
|
||||
and the possibility to execute commands. This includes [checksum](#checksum),
|
||||
and in some cases also [about](#about-command). The shell commands that
|
||||
must be executed may be different on different type of shells, and also
|
||||
quoting/escaping of file path arguments containing special characters may
|
||||
be different. Rclone therefore needs to know what type of shell it is,
|
||||
and if shell access is available at all.
|
||||
|
||||
Most servers run on some version of Unix, and then a basic Unix shell can
|
||||
be assumed, without further distinction. Windows 10, Server 2019, and later
|
||||
can also run a SSH server, which is a port of OpenSSH (see official
|
||||
[installation guide](https://docs.microsoft.com/en-us/windows-server/administration/openssh/openssh_install_firstuse)). On a Windows server the shell handling is different: Although it can also
|
||||
be set up to use a Unix type shell, e.g. Cygwin bash, the default is to
|
||||
use Windows Command Prompt (cmd.exe), and PowerShell is a recommended
|
||||
alternative. All of these have bahave differently, which rclone must handle.
|
||||
|
||||
Rclone tries to auto-detect what type of shell is used on the server,
|
||||
first time you access the SFTP remote. If a remote shell session is
|
||||
successfully created, it will look for indications that it is CMD or
|
||||
PowerShell, with fall-back to Unix if not something else is detected.
|
||||
If unable to even create a remote shell session, then shell command
|
||||
execution will be disabled entirely. The result is stored in the SFTP
|
||||
remote configuration, in option `shell_type`, so that the auto-detection
|
||||
only have to be performed once. If you manually set a value for this
|
||||
option before first run, the auto-detection will be skipped, and if
|
||||
you set a different value later this will override any existing.
|
||||
Value `none` can be set to avoid any attempts at executing shell
|
||||
commands, e.g. if this is not allowed on the server.
|
||||
|
||||
When the server is [rclone serve sftp](/commands/rclone_serve_sftp/),
|
||||
the rclone SFTP remote will detect this as a Unix type shell - even
|
||||
if it is running on Windows. This server does not actually have a shell,
|
||||
but it accepts input commands matching the specific ones that the
|
||||
SFTP backend relies on for Unix shells, e.g. `md5sum` and `df`. Also
|
||||
it handles the string escape rules used for Unix shell. Treating it
|
||||
as a Unix type shell from a SFTP remote will therefore always be
|
||||
correct, and support all features.
|
||||
|
||||
#### Shell access considerations
|
||||
|
||||
The shell type auto-detection logic, described above, means that
|
||||
by default rclone will try to run a shell command the first time
|
||||
a new sftp remote is accessed. If you configure a sftp remote
|
||||
without a config file, e.g. an [on the fly](/docs/#backend-path-to-dir])
|
||||
remote, rclone will have nowhere to store the result, and it
|
||||
will re-run the command on every access. To avoid this you should
|
||||
explicitely set the `shell_type` option to the correct value,
|
||||
or to `none` if you want to prevent rclone from executing any
|
||||
remote shell commands.
|
||||
|
||||
It is also important to note that, since the shell type decides
|
||||
how quoting and escaping of file paths used as command-line arguments
|
||||
are performed, configuring the wrong shell type may leave you exposed
|
||||
to command injection exploits. Make sure to confirm the auto-detected
|
||||
shell type, or explicitely set the shell type you know is correct,
|
||||
or disable shell access until you know.
|
||||
|
||||
### Checksum
|
||||
|
||||
SFTP does not natively support checksums (file hash), but rclone
|
||||
is able to use checksumming if the same login has shell access,
|
||||
and can execute remote commands. If there is a command that can
|
||||
calculate compatible checksums on the remote system, Rclone can
|
||||
then be configured to execute this whenever a checksum is needed,
|
||||
and read back the results. Currently MD5 and SHA-1 are supported.
|
||||
|
||||
Normally this requires an external utility being available on
|
||||
the server. By default rclone will try commands `md5sum`, `md5`
|
||||
and `rclone md5sum` for MD5 checksums, and the first one found usable
|
||||
will be picked. Same with `sha1sum`, `sha1` and `rclone sha1sum`
|
||||
commands for SHA-1 checksums. These utilities normally need to
|
||||
be in the remote's PATH to be found.
|
||||
|
||||
In some cases the shell itself is capable of calculating checksums.
|
||||
PowerShell is an example of such a shell. If rclone detects that the
|
||||
remote shell is PowerShell, which means it most probably is a
|
||||
Windows OpenSSH server, rclone will use a predefined script block
|
||||
to produce the checksums when no external checksum commands are found
|
||||
(see [shell access](#shell-access)). This assumes PowerShell version
|
||||
4.0 or newer.
|
||||
|
||||
The options `md5sum_command` and `sha1_command` can be used to customize
|
||||
the command to be executed for calculation of checksums. You can for
|
||||
example set a specific path to where md5sum and sha1sum executables
|
||||
are located, or use them to specify some other tools that print checksums
|
||||
in compatible format. The value can include command-line arguments,
|
||||
or even shell script blocks as with PowerShell. Rclone has subcommands
|
||||
[md5sum](/commands/rclone_md5sum/) and [sha1sum](/commands/rclone_sha1sum/)
|
||||
that use compatible format, which means if you have an rclone executable
|
||||
on the server it can be used. As mentioned above, they will be automatically
|
||||
picked up if found in PATH, but if not you can set something like
|
||||
`/path/to/rclone md5sum` as the value of option `md5sum_command` to
|
||||
make sure a specific executable is used.
|
||||
|
||||
Remote checksumming is recommended and enabled by default. First time
|
||||
rclone is using a SFTP remote, if options `md5sum_command` or `sha1_command`
|
||||
are not set, it will check if any of the default commands for each of them,
|
||||
as described above, can be used. The result will be saved in the remote
|
||||
configuration, so next time it will use the same. Value `none`
|
||||
will be set if none of the default commands could be used for a specific
|
||||
algorithm, and this algorithm will not be supported by the remote.
|
||||
|
||||
Disabling the checksumming may be required if you are connecting to SFTP servers
|
||||
which are not under your control, and to which the execution of remote shell
|
||||
commands is prohibited. Set the configuration option `disable_hashcheck`
|
||||
to `true` to disable checksumming entirely, or set `shell_type` to `none`
|
||||
to disable all functionality based on remote shell command execution.
|
||||
|
||||
### Modified time
|
||||
|
||||
Modified times are stored on the server to 1 second precision.
|
||||
|
|
@ -255,6 +368,20 @@ upload (for example, certain configurations of ProFTPd with mod_sftp). If you
|
|||
are using one of these servers, you can set the option `set_modtime = false` in
|
||||
your RClone backend configuration to disable this behaviour.
|
||||
|
||||
### About command
|
||||
|
||||
SFTP supports the [about](/commands/rclone_about/) command if the
|
||||
same login has access to a Unix shell, where the `df` command is
|
||||
available (e.g. in the remote's PATH). `about` will return the
|
||||
total space, free space, and used space on the remote for the disk
|
||||
of the specified path on the remote or, if not set, the disk of
|
||||
the root on the remote. `about` will fail if it does not have
|
||||
shell access or if `df` is not found.
|
||||
|
||||
If the server shell is PowerShell, probably with a Windows OpenSSH
|
||||
server, rclone supports `about` using a built-in shell command
|
||||
(see [shell access](#shell-access)).
|
||||
|
||||
{{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/sftp/sftp.go then run make backenddocs" >}}
|
||||
### Standard options
|
||||
|
||||
|
|
@ -637,25 +764,9 @@ Properties:
|
|||
|
||||
## Limitations
|
||||
|
||||
SFTP supports checksums if the same login has shell access and `md5sum`
|
||||
or `sha1sum` as well as `echo` are in the remote's PATH.
|
||||
This remote checksumming (file hashing) is recommended and enabled by default.
|
||||
Disabling the checksumming may be required if you are connecting to SFTP servers
|
||||
which are not under your control, and to which the execution of remote commands
|
||||
is prohibited. Set the configuration option `disable_hashcheck` to `true` to
|
||||
disable checksumming.
|
||||
|
||||
SFTP also supports `about` if the same login has shell
|
||||
access and `df` are in the remote's PATH. `about` will
|
||||
return the total space, free space, and used space on the remote
|
||||
for the disk of the specified path on the remote or, if not set,
|
||||
the disk of the root on the remote.
|
||||
`about` will fail if it does not have shell
|
||||
access or if `df` is not in the remote's PATH.
|
||||
|
||||
Note that some SFTP servers (e.g. Synology) the paths are different for
|
||||
SSH and SFTP so the hashes can't be calculated properly. For them
|
||||
using `disable_hashcheck` is a good idea.
|
||||
On some SFTP servers (e.g. Synology) the paths are different
|
||||
for SSH and SFTP so the hashes can't be calculated properly.
|
||||
For them using `disable_hashcheck` is a good idea.
|
||||
|
||||
The only ssh agent supported under Windows is Putty's pageant.
|
||||
|
||||
|
|
@ -670,11 +781,10 @@ SFTP isn't supported under plan9 until [this
|
|||
issue](https://github.com/pkg/sftp/issues/156) is fixed.
|
||||
|
||||
Note that since SFTP isn't HTTP based the following flags don't work
|
||||
with it: `--dump-headers`, `--dump-bodies`, `--dump-auth`
|
||||
with it: `--dump-headers`, `--dump-bodies`, `--dump-auth`.
|
||||
|
||||
Note that `--timeout` and `--contimeout` are both supported.
|
||||
|
||||
|
||||
## rsync.net {#rsync-net}
|
||||
|
||||
rsync.net is supported through the SFTP backend.
|
||||
|
|
|
|||
Loading…
Reference in New Issue