From c7a2719facc2afcc4617861063129927c60bf93e Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 21 Sep 2023 14:55:48 +0100 Subject: [PATCH] sftp: implement --sftp-copy-is-hardlink to server side copy as hardlink If the server does not support hardlinks then it falls back to normal copy. See: https://forum.rclone.org/t/sftp-remote-server-side-copy/41867 --- backend/sftp/sftp.go | 63 +++++++++++++ docs/content/overview.md | 112 +++++++++++------------ fstest/testserver/init.d/TestSFTPOpenssh | 1 + 3 files changed, 120 insertions(+), 56 deletions(-) diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go index 6b425c7d0..4508d780b 100644 --- a/backend/sftp/sftp.go +++ b/backend/sftp/sftp.go @@ -449,6 +449,26 @@ Example: myUser:myPass@localhost:9005 `, Advanced: true, + }, { + Name: "copy_is_hardlink", + Default: false, + Help: `Set to enable server side copies using hardlinks. + +The SFTP protocol does not define a copy command so normally server +side copies are not allowed with the sftp backend. + +However the SFTP protocol does support hardlinking, and if you enable +this flag then the sftp backend will support server side copies. These +will be implemented by doing a hardlink from the source to the +destination. + +Not all sftp servers support this. + +Note that hardlinking two files together will use no additional space +as the source and the destination will be the same file. + +This feature may be useful backups made with --copy-dest.`, + Advanced: true, }}, } fs.Register(fsi) @@ -490,6 +510,7 @@ type Options struct { HostKeyAlgorithms fs.SpaceSepList `config:"host_key_algorithms"` SSH fs.SpaceSepList `config:"ssh"` SocksProxy string `config:"socks_proxy"` + CopyIsHardlink bool `config:"copy_is_hardlink"` } // Fs stores the interface to the remote SFTP files @@ -1049,6 +1070,10 @@ func NewFsWithConnection(ctx context.Context, f *Fs, name string, root string, m SlowHash: true, PartialUploads: true, }).Fill(ctx, f) + if !opt.CopyIsHardlink { + // Disable server side copy unless --sftp-copy-is-hardlink is set + f.features.Copy = nil + } // Make a connection and pool it to return errors early c, err := f.getSftpConnection(ctx) if err != nil { @@ -1401,6 +1426,43 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, return dstObj, nil } +// Copy server side copies a remote sftp file object using hardlinks +func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { + if !f.opt.CopyIsHardlink { + return nil, fs.ErrorCantCopy + } + srcObj, ok := src.(*Object) + if !ok { + fs.Debugf(src, "Can't copy - not same remote type") + return nil, fs.ErrorCantCopy + } + err := f.mkParentDir(ctx, remote) + if err != nil { + return nil, fmt.Errorf("Copy mkParentDir failed: %w", err) + } + c, err := f.getSftpConnection(ctx) + if err != nil { + return nil, fmt.Errorf("Copy: %w", err) + } + srcPath, dstPath := srcObj.path(), path.Join(f.absRoot, remote) + err = c.sftpClient.Link(srcPath, dstPath) + f.putSftpConnection(&c, err) + if err != nil { + if sftpErr, ok := err.(*sftp.StatusError); ok { + if sftpErr.FxCode() == sftp.ErrSSHFxOpUnsupported { + // Remote doesn't support Link + return nil, fs.ErrorCantCopy + } + } + return nil, fmt.Errorf("Copy failed: %w", err) + } + dstObj, err := f.NewObject(ctx, remote) + if err != nil { + return nil, fmt.Errorf("Copy NewObject failed: %w", err) + } + return dstObj, nil +} + // DirMove moves src, srcRemote to this remote at dstRemote // using server-side move operations. // @@ -2120,6 +2182,7 @@ var ( _ fs.Fs = &Fs{} _ fs.PutStreamer = &Fs{} _ fs.Mover = &Fs{} + _ fs.Copier = &Fs{} _ fs.DirMover = &Fs{} _ fs.Abouter = &Fs{} _ fs.Shutdowner = &Fs{} diff --git a/docs/content/overview.md b/docs/content/overview.md index a17c74d3b..a49c6bd4b 100644 --- a/docs/content/overview.md +++ b/docs/content/overview.md @@ -62,8 +62,6 @@ Here is an overview of the major features of each cloud storage system. | Zoho WorkDrive | - | - | No | No | - | - | | The local filesystem | All | R/W | Depends | No | - | RWU | -### Notes - ¹ Dropbox supports [its own custom hash](https://www.dropbox.com/developers/reference/content-hash). This is an SHA256 sum of all the 4 MiB block SHA256s. @@ -469,66 +467,68 @@ upon backend-specific capabilities. | Name | Purge | Copy | Move | DirMove | CleanUp | ListR | StreamUpload | MultithreadUpload | LinkSharing | About | EmptyDir | | ---------------------------- |:-----:|:----:|:----:|:-------:|:-------:|:-----:|:------------:|:------------------|:------------:|:-----:|:--------:| -| 1Fichier | No | Yes | Yes | No | No | No | No | No | Yes | No | Yes | -| Akamai Netstorage | Yes | No | No | No | No | Yes | Yes | No | No | No | Yes | -| Amazon Drive | Yes | No | Yes | Yes | No | No | No | No | No | No | Yes | -| Amazon S3 (or S3 compatible) | No | Yes | No | No | Yes | Yes | Yes | Yes | Yes | No | No | -| Backblaze B2 | No | Yes | No | No | Yes | Yes | Yes | Yes | Yes | No | No | -| Box | Yes | Yes | Yes | Yes | Yes ‡‡ | No | Yes | No | Yes | Yes | Yes | -| Citrix ShareFile | Yes | Yes | Yes | Yes | No | No | No | No | No | No | Yes | -| Dropbox | Yes | Yes | Yes | Yes | No | No | Yes | No | Yes | Yes | Yes | -| Enterprise File Fabric | Yes | Yes | Yes | Yes | Yes | No | No | No | No | No | Yes | -| FTP | No | No | Yes | Yes | No | No | Yes | No | No | No | Yes | -| Google Cloud Storage | Yes | Yes | No | No | No | Yes | Yes | No | No | No | No | -| Google Drive | Yes | Yes | Yes | Yes | Yes | Yes | Yes | No | Yes | Yes | Yes | -| Google Photos | No | No | No | No | No | No | No | No | No | No | No | -| HDFS | Yes | No | Yes | Yes | No | No | Yes | No | No | Yes | Yes | -| HiDrive | Yes | Yes | Yes | Yes | No | No | Yes | No | No | No | Yes | -| HTTP | No | No | No | No | No | No | No | No | No | No | Yes | -| Internet Archive | No | Yes | No | No | Yes | Yes | No | No | Yes | Yes | No | -| Jottacloud | Yes | Yes | Yes | Yes | Yes | Yes | No | No | Yes | Yes | Yes | -| Koofr | Yes | Yes | Yes | Yes | No | No | Yes | No | Yes | Yes | Yes | -| Mail.ru Cloud | Yes | Yes | Yes | Yes | Yes | No | No | No | Yes | Yes | Yes | -| Mega | Yes | No | Yes | Yes | Yes | No | No | No | Yes | Yes | Yes | -| Memory | No | Yes | No | No | No | Yes | Yes | No | No | No | No | -| Microsoft Azure Blob Storage | Yes | Yes | No | No | No | Yes | Yes | Yes | No | No | No | -| Microsoft OneDrive | Yes | Yes | Yes | Yes | Yes | Yes | No | No | Yes | Yes | Yes | -| OpenDrive | Yes | Yes | Yes | Yes | No | No | No | No | No | No | Yes | -| OpenStack Swift | Yes † | Yes | No | No | No | Yes | Yes | No | No | Yes | No | -| Oracle Object Storage | No | Yes | No | No | Yes | Yes | Yes | Yes | No | No | No | -| pCloud | Yes | Yes | Yes | Yes | Yes | No | No | No | Yes | Yes | Yes | -| PikPak | Yes | Yes | Yes | Yes | Yes | No | No | No | Yes | Yes | Yes | -| premiumize.me | Yes | No | Yes | Yes | No | No | No | No | Yes | Yes | Yes | -| put.io | Yes | No | Yes | Yes | Yes | No | Yes | No | No | Yes | Yes | -| Proton Drive | Yes | No | Yes | Yes | Yes | No | No | No | No | Yes | Yes | -| QingStor | No | Yes | No | No | Yes | Yes | No | No | No | No | No | -| Quatrix by Maytech | Yes | Yes | Yes | Yes | No | No | No | No | No | Yes | Yes | -| Seafile | Yes | Yes | Yes | Yes | Yes | Yes | Yes | No | Yes | Yes | Yes | -| SFTP | No | No | Yes | Yes | No | No | Yes | No | No | Yes | Yes | -| Sia | No | No | No | No | No | No | Yes | No | No | No | Yes | -| SMB | No | No | Yes | Yes | No | No | Yes | Yes | No | No | Yes | -| SugarSync | Yes | Yes | Yes | Yes | No | No | Yes | No | Yes | No | Yes | -| Storj | Yes ☨ | Yes | Yes | No | No | Yes | Yes | No | Yes | No | No | -| Uptobox | No | Yes | Yes | Yes | No | No | No | No | No | No | No | -| WebDAV | Yes | Yes | Yes | Yes | No | No | Yes ‡ | No | No | Yes | Yes | -| Yandex Disk | Yes | Yes | Yes | Yes | Yes | No | Yes | No | Yes | Yes | Yes | -| Zoho WorkDrive | Yes | Yes | Yes | Yes | No | No | No | No | No | Yes | Yes | -| The local filesystem | Yes | No | Yes | Yes | No | No | Yes | Yes | No | Yes | Yes | +| 1Fichier | No | Yes | Yes | No | No | No | No | No | Yes | No | Yes | +| Akamai Netstorage | Yes | No | No | No | No | Yes | Yes | No | No | No | Yes | +| Amazon Drive | Yes | No | Yes | Yes | No | No | No | No | No | No | Yes | +| Amazon S3 (or S3 compatible) | No | Yes | No | No | Yes | Yes | Yes | Yes | Yes | No | No | +| Backblaze B2 | No | Yes | No | No | Yes | Yes | Yes | Yes | Yes | No | No | +| Box | Yes | Yes | Yes | Yes | Yes | No | Yes | No | Yes | Yes | Yes | +| Citrix ShareFile | Yes | Yes | Yes | Yes | No | No | No | No | No | No | Yes | +| Dropbox | Yes | Yes | Yes | Yes | No | No | Yes | No | Yes | Yes | Yes | +| Enterprise File Fabric | Yes | Yes | Yes | Yes | Yes | No | No | No | No | No | Yes | +| FTP | No | No | Yes | Yes | No | No | Yes | No | No | No | Yes | +| Google Cloud Storage | Yes | Yes | No | No | No | Yes | Yes | No | No | No | No | +| Google Drive | Yes | Yes | Yes | Yes | Yes | Yes | Yes | No | Yes | Yes | Yes | +| Google Photos | No | No | No | No | No | No | No | No | No | No | No | +| HDFS | Yes | No | Yes | Yes | No | No | Yes | No | No | Yes | Yes | +| HiDrive | Yes | Yes | Yes | Yes | No | No | Yes | No | No | No | Yes | +| HTTP | No | No | No | No | No | No | No | No | No | No | Yes | +| Internet Archive | No | Yes | No | No | Yes | Yes | No | No | Yes | Yes | No | +| Jottacloud | Yes | Yes | Yes | Yes | Yes | Yes | No | No | Yes | Yes | Yes | +| Koofr | Yes | Yes | Yes | Yes | No | No | Yes | No | Yes | Yes | Yes | +| Mail.ru Cloud | Yes | Yes | Yes | Yes | Yes | No | No | No | Yes | Yes | Yes | +| Mega | Yes | No | Yes | Yes | Yes | No | No | No | Yes | Yes | Yes | +| Memory | No | Yes | No | No | No | Yes | Yes | No | No | No | No | +| Microsoft Azure Blob Storage | Yes | Yes | No | No | No | Yes | Yes | Yes | No | No | No | +| Microsoft OneDrive | Yes | Yes | Yes | Yes | Yes | Yes | No | No | Yes | Yes | Yes | +| OpenDrive | Yes | Yes | Yes | Yes | No | No | No | No | No | No | Yes | +| OpenStack Swift | Yes ¹ | Yes | No | No | No | Yes | Yes | No | No | Yes | No | +| Oracle Object Storage | No | Yes | No | No | Yes | Yes | Yes | Yes | No | No | No | +| pCloud | Yes | Yes | Yes | Yes | Yes | No | No | No | Yes | Yes | Yes | +| PikPak | Yes | Yes | Yes | Yes | Yes | No | No | No | Yes | Yes | Yes | +| premiumize.me | Yes | No | Yes | Yes | No | No | No | No | Yes | Yes | Yes | +| put.io | Yes | No | Yes | Yes | Yes | No | Yes | No | No | Yes | Yes | +| Proton Drive | Yes | No | Yes | Yes | Yes | No | No | No | No | Yes | Yes | +| QingStor | No | Yes | No | No | Yes | Yes | No | No | No | No | No | +| Quatrix by Maytech | Yes | Yes | Yes | Yes | No | No | No | No | No | Yes | Yes | +| Seafile | Yes | Yes | Yes | Yes | Yes | Yes | Yes | No | Yes | Yes | Yes | +| SFTP | No | Yes ⁴| Yes | Yes | No | No | Yes | No | No | Yes | Yes | +| Sia | No | No | No | No | No | No | Yes | No | No | No | Yes | +| SMB | No | No | Yes | Yes | No | No | Yes | Yes | No | No | Yes | +| SugarSync | Yes | Yes | Yes | Yes | No | No | Yes | No | Yes | No | Yes | +| Storj | Yes ² | Yes | Yes | No | No | Yes | Yes | No | Yes | No | No | +| Uptobox | No | Yes | Yes | Yes | No | No | No | No | No | No | No | +| WebDAV | Yes | Yes | Yes | Yes | No | No | Yes ³ | No | No | Yes | Yes | +| Yandex Disk | Yes | Yes | Yes | Yes | Yes | No | Yes | No | Yes | Yes | Yes | +| Zoho WorkDrive | Yes | Yes | Yes | Yes | No | No | No | No | No | Yes | Yes | +| The local filesystem | Yes | No | Yes | Yes | No | No | Yes | Yes | No | Yes | Yes | + +¹ Note Swift implements this in order to delete directory markers but +it doesn't actually have a quicker way of deleting files other than +deleting them individually. + +² Storj implements this efficiently only for entire buckets. If +purging a directory inside a bucket, files are deleted individually. + +³ StreamUpload is not supported with Nextcloud + +⁴ Use the `--sftp-copy-is-hardlink` flag to enable. ### Purge ### This deletes a directory quicker than just deleting all the files in the directory. -† Note Swift implements this in order to delete directory markers but -they don't actually have a quicker way of deleting files other than -deleting them individually. - -☨ Storj implements this efficiently only for entire buckets. If -purging a directory inside a bucket, files are deleted individually. - -‡ StreamUpload is not supported with Nextcloud - ### Copy ### Used when copying an object to and from the same remote. This known diff --git a/fstest/testserver/init.d/TestSFTPOpenssh b/fstest/testserver/init.d/TestSFTPOpenssh index ef29095d1..4eb14ac9a 100755 --- a/fstest/testserver/init.d/TestSFTPOpenssh +++ b/fstest/testserver/init.d/TestSFTPOpenssh @@ -16,6 +16,7 @@ start() { echo host=$(docker_ip) echo user=$USER echo pass=$(rclone obscure $PASS) + echo copy_is_hardlink=true echo _connect=$(docker_ip):22 }