community
/
rclone
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

sftp: implement --sftp-copy-is-hardlink to server side copy as hardlink 

If the server does not support hardlinks then it falls back to normal
copy.

See: https://forum.rclone.org/t/sftp-remote-server-side-copy/41867
This commit is contained in:
Nick Craig-Wood 2023-09-21 14:55:48 +01:00
parent c190b9b14f
commit c7a2719fac
3 changed files with 120 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
backend/sftp/sftp.go
View File

@ -449,6 +449,26 @@ Example:
myUser:myPass@localhost:9005 myUser:myPass@localhost:9005
`, `,
Advanced: true, Advanced: true,
}, {
Name: "copy_is_hardlink",
Default: false,
Help: `Set to enable server side copies using hardlinks.
The SFTP protocol does not define a copy command so normally server
side copies are not allowed with the sftp backend.
However the SFTP protocol does support hardlinking, and if you enable
this flag then the sftp backend will support server side copies. These
will be implemented by doing a hardlink from the source to the
destination.
Not all sftp servers support this.
Note that hardlinking two files together will use no additional space
as the source and the destination will be the same file.
This feature may be useful backups made with --copy-dest.`,
Advanced: true,
}}, }},
} }
fs.Register(fsi) fs.Register(fsi)
@ -490,6 +510,7 @@ type Options struct {
HostKeyAlgorithms fs.SpaceSepList `config:"host_key_algorithms"` HostKeyAlgorithms fs.SpaceSepList `config:"host_key_algorithms"`
SSH fs.SpaceSepList `config:"ssh"` SSH fs.SpaceSepList `config:"ssh"`
SocksProxy string `config:"socks_proxy"` SocksProxy string `config:"socks_proxy"`
CopyIsHardlink bool `config:"copy_is_hardlink"`
} }
// Fs stores the interface to the remote SFTP files // Fs stores the interface to the remote SFTP files
@ -1049,6 +1070,10 @@ func NewFsWithConnection(ctx context.Context, f *Fs, name string, root string, m
SlowHash: true, SlowHash: true,
PartialUploads: true, PartialUploads: true,
}).Fill(ctx, f) }).Fill(ctx, f)
if !opt.CopyIsHardlink {
// Disable server side copy unless --sftp-copy-is-hardlink is set
f.features.Copy = nil
}
// Make a connection and pool it to return errors early // Make a connection and pool it to return errors early
c, err := f.getSftpConnection(ctx) c, err := f.getSftpConnection(ctx)
if err != nil { if err != nil {
@ -1401,6 +1426,43 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
return dstObj, nil return dstObj, nil
} }
// Copy server side copies a remote sftp file object using hardlinks
func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
if !f.opt.CopyIsHardlink {
return nil, fs.ErrorCantCopy
}
srcObj, ok := src.(*Object)
if !ok {
fs.Debugf(src, "Can't copy - not same remote type")
return nil, fs.ErrorCantCopy
}
err := f.mkParentDir(ctx, remote)
if err != nil {
return nil, fmt.Errorf("Copy mkParentDir failed: %w", err)
}
c, err := f.getSftpConnection(ctx)
if err != nil {
return nil, fmt.Errorf("Copy: %w", err)
}
srcPath, dstPath := srcObj.path(), path.Join(f.absRoot, remote)
err = c.sftpClient.Link(srcPath, dstPath)
f.putSftpConnection(&c, err)
if err != nil {
if sftpErr, ok := err.(*sftp.StatusError); ok {
if sftpErr.FxCode() == sftp.ErrSSHFxOpUnsupported {
// Remote doesn't support Link
return nil, fs.ErrorCantCopy
}
}
return nil, fmt.Errorf("Copy failed: %w", err)
}
dstObj, err := f.NewObject(ctx, remote)
if err != nil {
return nil, fmt.Errorf("Copy NewObject failed: %w", err)
}
return dstObj, nil
}
// DirMove moves src, srcRemote to this remote at dstRemote // DirMove moves src, srcRemote to this remote at dstRemote
// using server-side move operations. // using server-side move operations.
// //
@ -2120,6 +2182,7 @@ var (
_ fs.Fs = &Fs{} _ fs.Fs = &Fs{}
_ fs.PutStreamer = &Fs{} _ fs.PutStreamer = &Fs{}
_ fs.Mover = &Fs{} _ fs.Mover = &Fs{}
_ fs.Copier = &Fs{}
_ fs.DirMover = &Fs{} _ fs.DirMover = &Fs{}
_ fs.Abouter = &Fs{} _ fs.Abouter = &Fs{}
_ fs.Shutdowner = &Fs{} _ fs.Shutdowner = &Fs{}

112
docs/content/overview.md
View File

@ -62,8 +62,6 @@ Here is an overview of the major features of each cloud storage system.
| Zoho WorkDrive | - | - | No | No | - | - | | Zoho WorkDrive | - | - | No | No | - | - |
| The local filesystem | All | R/W | Depends | No | - | RWU | | The local filesystem | All | R/W | Depends | No | - | RWU |
### Notes
¹ Dropbox supports [its own custom ¹ Dropbox supports [its own custom
hash](https://www.dropbox.com/developers/reference/content-hash). hash](https://www.dropbox.com/developers/reference/content-hash).
This is an SHA256 sum of all the 4 MiB block SHA256s. This is an SHA256 sum of all the 4 MiB block SHA256s.
@ -469,66 +467,68 @@ upon backend-specific capabilities.
| Name | Purge | Copy | Move | DirMove | CleanUp | ListR | StreamUpload | MultithreadUpload | LinkSharing | About | EmptyDir | | Name | Purge | Copy | Move | DirMove | CleanUp | ListR | StreamUpload | MultithreadUpload | LinkSharing | About | EmptyDir |
| ---------------------------- |:-----:|:----:|:----:|:-------:|:-------:|:-----:|:------------:|:------------------|:------------:|:-----:|:--------:| | ---------------------------- |:-----:|:----:|:----:|:-------:|:-------:|:-----:|:------------:|:------------------|:------------:|:-----:|:--------:|
| 1Fichier | No | Yes | Yes | No | No | No | No | No | Yes | No | Yes | | 1Fichier | No | Yes | Yes | No | No | No | No | No | Yes | No | Yes |
| Akamai Netstorage | Yes | No | No | No | No | Yes | Yes | No | No | No | Yes | | Akamai Netstorage | Yes | No | No | No | No | Yes | Yes | No | No | No | Yes |
| Amazon Drive | Yes | No | Yes | Yes | No | No | No | No | No | No | Yes | | Amazon Drive | Yes | No | Yes | Yes | No | No | No | No | No | No | Yes |
| Amazon S3 (or S3 compatible) | No | Yes | No | No | Yes | Yes | Yes | Yes | Yes | No | No | | Amazon S3 (or S3 compatible) | No | Yes | No | No | Yes | Yes | Yes | Yes | Yes | No | No |
| Backblaze B2 | No | Yes | No | No | Yes | Yes | Yes | Yes | Yes | No | No | | Backblaze B2 | No | Yes | No | No | Yes | Yes | Yes | Yes | Yes | No | No |
| Box | Yes | Yes | Yes | Yes | Yes ‡‡ | No | Yes | No | Yes | Yes | Yes | | Box | Yes | Yes | Yes | Yes | Yes | No | Yes | No | Yes | Yes | Yes |
| Citrix ShareFile | Yes | Yes | Yes | Yes | No | No | No | No | No | No | Yes | | Citrix ShareFile | Yes | Yes | Yes | Yes | No | No | No | No | No | No | Yes |
| Dropbox | Yes | Yes | Yes | Yes | No | No | Yes | No | Yes | Yes | Yes | | Dropbox | Yes | Yes | Yes | Yes | No | No | Yes | No | Yes | Yes | Yes |
| Enterprise File Fabric | Yes | Yes | Yes | Yes | Yes | No | No | No | No | No | Yes | | Enterprise File Fabric | Yes | Yes | Yes | Yes | Yes | No | No | No | No | No | Yes |
| FTP | No | No | Yes | Yes | No | No | Yes | No | No | No | Yes | | FTP | No | No | Yes | Yes | No | No | Yes | No | No | No | Yes |
| Google Cloud Storage | Yes | Yes | No | No | No | Yes | Yes | No | No | No | No | | Google Cloud Storage | Yes | Yes | No | No | No | Yes | Yes | No | No | No | No |
| Google Drive | Yes | Yes | Yes | Yes | Yes | Yes | Yes | No | Yes | Yes | Yes | | Google Drive | Yes | Yes | Yes | Yes | Yes | Yes | Yes | No | Yes | Yes | Yes |
| Google Photos | No | No | No | No | No | No | No | No | No | No | No | | Google Photos | No | No | No | No | No | No | No | No | No | No | No |
| HDFS | Yes | No | Yes | Yes | No | No | Yes | No | No | Yes | Yes | | HDFS | Yes | No | Yes | Yes | No | No | Yes | No | No | Yes | Yes |
| HiDrive | Yes | Yes | Yes | Yes | No | No | Yes | No | No | No | Yes | | HiDrive | Yes | Yes | Yes | Yes | No | No | Yes | No | No | No | Yes |
| HTTP | No | No | No | No | No | No | No | No | No | No | Yes | | HTTP | No | No | No | No | No | No | No | No | No | No | Yes |
| Internet Archive | No | Yes | No | No | Yes | Yes | No | No | Yes | Yes | No | | Internet Archive | No | Yes | No | No | Yes | Yes | No | No | Yes | Yes | No |
| Jottacloud | Yes | Yes | Yes | Yes | Yes | Yes | No | No | Yes | Yes | Yes | | Jottacloud | Yes | Yes | Yes | Yes | Yes | Yes | No | No | Yes | Yes | Yes |
| Koofr | Yes | Yes | Yes | Yes | No | No | Yes | No | Yes | Yes | Yes | | Koofr | Yes | Yes | Yes | Yes | No | No | Yes | No | Yes | Yes | Yes |
| Mail.ru Cloud | Yes | Yes | Yes | Yes | Yes | No | No | No | Yes | Yes | Yes | | Mail.ru Cloud | Yes | Yes | Yes | Yes | Yes | No | No | No | Yes | Yes | Yes |
| Mega | Yes | No | Yes | Yes | Yes | No | No | No | Yes | Yes | Yes | | Mega | Yes | No | Yes | Yes | Yes | No | No | No | Yes | Yes | Yes |
| Memory | No | Yes | No | No | No | Yes | Yes | No | No | No | No | | Memory | No | Yes | No | No | No | Yes | Yes | No | No | No | No |
| Microsoft Azure Blob Storage | Yes | Yes | No | No | No | Yes | Yes | Yes | No | No | No | | Microsoft Azure Blob Storage | Yes | Yes | No | No | No | Yes | Yes | Yes | No | No | No |
| Microsoft OneDrive | Yes | Yes | Yes | Yes | Yes | Yes | No | No | Yes | Yes | Yes | | Microsoft OneDrive | Yes | Yes | Yes | Yes | Yes | Yes | No | No | Yes | Yes | Yes |
| OpenDrive | Yes | Yes | Yes | Yes | No | No | No | No | No | No | Yes | | OpenDrive | Yes | Yes | Yes | Yes | No | No | No | No | No | No | Yes |
| OpenStack Swift | Yes † | Yes | No | No | No | Yes | Yes | No | No | Yes | No | | OpenStack Swift | Yes ¹ | Yes | No | No | No | Yes | Yes | No | No | Yes | No |
| Oracle Object Storage | No | Yes | No | No | Yes | Yes | Yes | Yes | No | No | No | | Oracle Object Storage | No | Yes | No | No | Yes | Yes | Yes | Yes | No | No | No |
| pCloud | Yes | Yes | Yes | Yes | Yes | No | No | No | Yes | Yes | Yes | | pCloud | Yes | Yes | Yes | Yes | Yes | No | No | No | Yes | Yes | Yes |
| PikPak | Yes | Yes | Yes | Yes | Yes | No | No | No | Yes | Yes | Yes | | PikPak | Yes | Yes | Yes | Yes | Yes | No | No | No | Yes | Yes | Yes |
| premiumize.me | Yes | No | Yes | Yes | No | No | No | No | Yes | Yes | Yes | | premiumize.me | Yes | No | Yes | Yes | No | No | No | No | Yes | Yes | Yes |
| put.io | Yes | No | Yes | Yes | Yes | No | Yes | No | No | Yes | Yes | | put.io | Yes | No | Yes | Yes | Yes | No | Yes | No | No | Yes | Yes |
| Proton Drive | Yes | No | Yes | Yes | Yes | No | No | No | No | Yes | Yes | | Proton Drive | Yes | No | Yes | Yes | Yes | No | No | No | No | Yes | Yes |
| QingStor | No | Yes | No | No | Yes | Yes | No | No | No | No | No | | QingStor | No | Yes | No | No | Yes | Yes | No | No | No | No | No |
| Quatrix by Maytech | Yes | Yes | Yes | Yes | No | No | No | No | No | Yes | Yes | | Quatrix by Maytech | Yes | Yes | Yes | Yes | No | No | No | No | No | Yes | Yes |
| Seafile | Yes | Yes | Yes | Yes | Yes | Yes | Yes | No | Yes | Yes | Yes | | Seafile | Yes | Yes | Yes | Yes | Yes | Yes | Yes | No | Yes | Yes | Yes |
| SFTP | No | No | Yes | Yes | No | No | Yes | No | No | Yes | Yes | | SFTP | No | Yes ⁴| Yes | Yes | No | No | Yes | No | No | Yes | Yes |
| Sia | No | No | No | No | No | No | Yes | No | No | No | Yes | | Sia | No | No | No | No | No | No | Yes | No | No | No | Yes |
| SMB | No | No | Yes | Yes | No | No | Yes | Yes | No | No | Yes | | SMB | No | No | Yes | Yes | No | No | Yes | Yes | No | No | Yes |
| SugarSync | Yes | Yes | Yes | Yes | No | No | Yes | No | Yes | No | Yes | | SugarSync | Yes | Yes | Yes | Yes | No | No | Yes | No | Yes | No | Yes |
| Storj | Yes ☨ | Yes | Yes | No | No | Yes | Yes | No | Yes | No | No | | Storj | Yes ² | Yes | Yes | No | No | Yes | Yes | No | Yes | No | No |
| Uptobox | No | Yes | Yes | Yes | No | No | No | No | No | No | No | | Uptobox | No | Yes | Yes | Yes | No | No | No | No | No | No | No |
| WebDAV | Yes | Yes | Yes | Yes | No | No | Yes ‡ | No | No | Yes | Yes | | WebDAV | Yes | Yes | Yes | Yes | No | No | Yes ³ | No | No | Yes | Yes |
| Yandex Disk | Yes | Yes | Yes | Yes | Yes | No | Yes | No | Yes | Yes | Yes | | Yandex Disk | Yes | Yes | Yes | Yes | Yes | No | Yes | No | Yes | Yes | Yes |
| Zoho WorkDrive | Yes | Yes | Yes | Yes | No | No | No | No | No | Yes | Yes | | Zoho WorkDrive | Yes | Yes | Yes | Yes | No | No | No | No | No | Yes | Yes |
| The local filesystem | Yes | No | Yes | Yes | No | No | Yes | Yes | No | Yes | Yes | | The local filesystem | Yes | No | Yes | Yes | No | No | Yes | Yes | No | Yes | Yes |
¹ Note Swift implements this in order to delete directory markers but
it doesn't actually have a quicker way of deleting files other than
deleting them individually.
² Storj implements this efficiently only for entire buckets. If
purging a directory inside a bucket, files are deleted individually.
³ StreamUpload is not supported with Nextcloud
⁴ Use the `--sftp-copy-is-hardlink` flag to enable.
### Purge ### ### Purge ###
This deletes a directory quicker than just deleting all the files in This deletes a directory quicker than just deleting all the files in
the directory. the directory.
† Note Swift implements this in order to delete directory markers but
they don't actually have a quicker way of deleting files other than
deleting them individually.
☨ Storj implements this efficiently only for entire buckets. If
purging a directory inside a bucket, files are deleted individually.
‡ StreamUpload is not supported with Nextcloud
### Copy ### ### Copy ###
Used when copying an object to and from the same remote. This known Used when copying an object to and from the same remote. This known

1
fstest/testserver/init.d/TestSFTPOpenssh
View File

@ -16,6 +16,7 @@ start() {
echo host=$(docker_ip) echo host=$(docker_ip)
echo user=$USER echo user=$USER
echo pass=$(rclone obscure $PASS) echo pass=$(rclone obscure $PASS)
echo copy_is_hardlink=true
echo _connect=$(docker_ip):22 echo _connect=$(docker_ip):22
} }