sftp: implement --sftp-copy-is-hardlink to server side copy as hardlink
If the server does not support hardlinks then it falls back to normal copy. See: https://forum.rclone.org/t/sftp-remote-server-side-copy/41867
This commit is contained in:
Nick Craig-Wood
parent
c190b9b14f
commit
c7a2719fac
|
|
@ -449,6 +449,26 @@ Example:
|
|||
myUser:myPass@localhost:9005
|
||||
`,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "copy_is_hardlink",
|
||||
Default: false,
|
||||
Help: `Set to enable server side copies using hardlinks.
|
||||
|
||||
The SFTP protocol does not define a copy command so normally server
|
||||
side copies are not allowed with the sftp backend.
|
||||
|
||||
However the SFTP protocol does support hardlinking, and if you enable
|
||||
this flag then the sftp backend will support server side copies. These
|
||||
will be implemented by doing a hardlink from the source to the
|
||||
destination.
|
||||
|
||||
Not all sftp servers support this.
|
||||
|
||||
Note that hardlinking two files together will use no additional space
|
||||
as the source and the destination will be the same file.
|
||||
|
||||
This feature may be useful backups made with --copy-dest.`,
|
||||
Advanced: true,
|
||||
}},
|
||||
}
|
||||
fs.Register(fsi)
|
||||
|
|
@ -490,6 +510,7 @@ type Options struct {
|
|||
HostKeyAlgorithms fs.SpaceSepList `config:"host_key_algorithms"`
|
||||
SSH fs.SpaceSepList `config:"ssh"`
|
||||
SocksProxy string `config:"socks_proxy"`
|
||||
CopyIsHardlink bool `config:"copy_is_hardlink"`
|
||||
}
|
||||
|
||||
// Fs stores the interface to the remote SFTP files
|
||||
|
|
@ -1049,6 +1070,10 @@ func NewFsWithConnection(ctx context.Context, f *Fs, name string, root string, m
|
|||
SlowHash: true,
|
||||
PartialUploads: true,
|
||||
}).Fill(ctx, f)
|
||||
if !opt.CopyIsHardlink {
|
||||
// Disable server side copy unless --sftp-copy-is-hardlink is set
|
||||
f.features.Copy = nil
|
||||
}
|
||||
// Make a connection and pool it to return errors early
|
||||
c, err := f.getSftpConnection(ctx)
|
||||
if err != nil {
|
||||
|
|
@ -1401,6 +1426,43 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
|||
return dstObj, nil
|
||||
}
|
||||
|
||||
// Copy server side copies a remote sftp file object using hardlinks
|
||||
func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
|
||||
if !f.opt.CopyIsHardlink {
|
||||
return nil, fs.ErrorCantCopy
|
||||
}
|
||||
srcObj, ok := src.(*Object)
|
||||
if !ok {
|
||||
fs.Debugf(src, "Can't copy - not same remote type")
|
||||
return nil, fs.ErrorCantCopy
|
||||
}
|
||||
err := f.mkParentDir(ctx, remote)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Copy mkParentDir failed: %w", err)
|
||||
}
|
||||
c, err := f.getSftpConnection(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Copy: %w", err)
|
||||
}
|
||||
srcPath, dstPath := srcObj.path(), path.Join(f.absRoot, remote)
|
||||
err = c.sftpClient.Link(srcPath, dstPath)
|
||||
f.putSftpConnection(&c, err)
|
||||
if err != nil {
|
||||
if sftpErr, ok := err.(*sftp.StatusError); ok {
|
||||
if sftpErr.FxCode() == sftp.ErrSSHFxOpUnsupported {
|
||||
// Remote doesn't support Link
|
||||
return nil, fs.ErrorCantCopy
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("Copy failed: %w", err)
|
||||
}
|
||||
dstObj, err := f.NewObject(ctx, remote)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Copy NewObject failed: %w", err)
|
||||
}
|
||||
return dstObj, nil
|
||||
}
|
||||
|
||||
// DirMove moves src, srcRemote to this remote at dstRemote
|
||||
// using server-side move operations.
|
||||
//
|
||||
|
|
@ -2120,6 +2182,7 @@ var (
|
|||
_ fs.Fs = &Fs{}
|
||||
_ fs.PutStreamer = &Fs{}
|
||||
_ fs.Mover = &Fs{}
|
||||
_ fs.Copier = &Fs{}
|
||||
_ fs.DirMover = &Fs{}
|
||||
_ fs.Abouter = &Fs{}
|
||||
_ fs.Shutdowner = &Fs{}
|
||||
|
|
|
|||
|
|
@ -62,8 +62,6 @@ Here is an overview of the major features of each cloud storage system.
|
|||
| Zoho WorkDrive | - | - | No | No | - | - |
|
||||
| The local filesystem | All | R/W | Depends | No | - | RWU |
|
||||
|
||||
### Notes
|
||||
|
||||
¹ Dropbox supports [its own custom
|
||||
hash](https://www.dropbox.com/developers/reference/content-hash).
|
||||
This is an SHA256 sum of all the 4 MiB block SHA256s.
|
||||
|
|
@ -469,66 +467,68 @@ upon backend-specific capabilities.
|
|||
|
||||
| Name | Purge | Copy | Move | DirMove | CleanUp | ListR | StreamUpload | MultithreadUpload | LinkSharing | About | EmptyDir |
|
||||
| ---------------------------- |:-----:|:----:|:----:|:-------:|:-------:|:-----:|:------------:|:------------------|:------------:|:-----:|:--------:|
|
||||
| 1Fichier | No | Yes | Yes | No | No | No | No | No | Yes | No | Yes |
|
||||
| Akamai Netstorage | Yes | No | No | No | No | Yes | Yes | No | No | No | Yes |
|
||||
| Amazon Drive | Yes | No | Yes | Yes | No | No | No | No | No | No | Yes |
|
||||
| Amazon S3 (or S3 compatible) | No | Yes | No | No | Yes | Yes | Yes | Yes | Yes | No | No |
|
||||
| Backblaze B2 | No | Yes | No | No | Yes | Yes | Yes | Yes | Yes | No | No |
|
||||
| Box | Yes | Yes | Yes | Yes | Yes ‡‡ | No | Yes | No | Yes | Yes | Yes |
|
||||
| Citrix ShareFile | Yes | Yes | Yes | Yes | No | No | No | No | No | No | Yes |
|
||||
| Dropbox | Yes | Yes | Yes | Yes | No | No | Yes | No | Yes | Yes | Yes |
|
||||
| Enterprise File Fabric | Yes | Yes | Yes | Yes | Yes | No | No | No | No | No | Yes |
|
||||
| FTP | No | No | Yes | Yes | No | No | Yes | No | No | No | Yes |
|
||||
| Google Cloud Storage | Yes | Yes | No | No | No | Yes | Yes | No | No | No | No |
|
||||
| Google Drive | Yes | Yes | Yes | Yes | Yes | Yes | Yes | No | Yes | Yes | Yes |
|
||||
| Google Photos | No | No | No | No | No | No | No | No | No | No | No |
|
||||
| HDFS | Yes | No | Yes | Yes | No | No | Yes | No | No | Yes | Yes |
|
||||
| HiDrive | Yes | Yes | Yes | Yes | No | No | Yes | No | No | No | Yes |
|
||||
| HTTP | No | No | No | No | No | No | No | No | No | No | Yes |
|
||||
| Internet Archive | No | Yes | No | No | Yes | Yes | No | No | Yes | Yes | No |
|
||||
| Jottacloud | Yes | Yes | Yes | Yes | Yes | Yes | No | No | Yes | Yes | Yes |
|
||||
| Koofr | Yes | Yes | Yes | Yes | No | No | Yes | No | Yes | Yes | Yes |
|
||||
| Mail.ru Cloud | Yes | Yes | Yes | Yes | Yes | No | No | No | Yes | Yes | Yes |
|
||||
| Mega | Yes | No | Yes | Yes | Yes | No | No | No | Yes | Yes | Yes |
|
||||
| Memory | No | Yes | No | No | No | Yes | Yes | No | No | No | No |
|
||||
| Microsoft Azure Blob Storage | Yes | Yes | No | No | No | Yes | Yes | Yes | No | No | No |
|
||||
| Microsoft OneDrive | Yes | Yes | Yes | Yes | Yes | Yes | No | No | Yes | Yes | Yes |
|
||||
| OpenDrive | Yes | Yes | Yes | Yes | No | No | No | No | No | No | Yes |
|
||||
| OpenStack Swift | Yes † | Yes | No | No | No | Yes | Yes | No | No | Yes | No |
|
||||
| Oracle Object Storage | No | Yes | No | No | Yes | Yes | Yes | Yes | No | No | No |
|
||||
| pCloud | Yes | Yes | Yes | Yes | Yes | No | No | No | Yes | Yes | Yes |
|
||||
| PikPak | Yes | Yes | Yes | Yes | Yes | No | No | No | Yes | Yes | Yes |
|
||||
| premiumize.me | Yes | No | Yes | Yes | No | No | No | No | Yes | Yes | Yes |
|
||||
| put.io | Yes | No | Yes | Yes | Yes | No | Yes | No | No | Yes | Yes |
|
||||
| Proton Drive | Yes | No | Yes | Yes | Yes | No | No | No | No | Yes | Yes |
|
||||
| QingStor | No | Yes | No | No | Yes | Yes | No | No | No | No | No |
|
||||
| Quatrix by Maytech | Yes | Yes | Yes | Yes | No | No | No | No | No | Yes | Yes |
|
||||
| Seafile | Yes | Yes | Yes | Yes | Yes | Yes | Yes | No | Yes | Yes | Yes |
|
||||
| SFTP | No | No | Yes | Yes | No | No | Yes | No | No | Yes | Yes |
|
||||
| Sia | No | No | No | No | No | No | Yes | No | No | No | Yes |
|
||||
| SMB | No | No | Yes | Yes | No | No | Yes | Yes | No | No | Yes |
|
||||
| SugarSync | Yes | Yes | Yes | Yes | No | No | Yes | No | Yes | No | Yes |
|
||||
| Storj | Yes ☨ | Yes | Yes | No | No | Yes | Yes | No | Yes | No | No |
|
||||
| Uptobox | No | Yes | Yes | Yes | No | No | No | No | No | No | No |
|
||||
| WebDAV | Yes | Yes | Yes | Yes | No | No | Yes ‡ | No | No | Yes | Yes |
|
||||
| Yandex Disk | Yes | Yes | Yes | Yes | Yes | No | Yes | No | Yes | Yes | Yes |
|
||||
| Zoho WorkDrive | Yes | Yes | Yes | Yes | No | No | No | No | No | Yes | Yes |
|
||||
| The local filesystem | Yes | No | Yes | Yes | No | No | Yes | Yes | No | Yes | Yes |
|
||||
| 1Fichier | No | Yes | Yes | No | No | No | No | No | Yes | No | Yes |
|
||||
| Akamai Netstorage | Yes | No | No | No | No | Yes | Yes | No | No | No | Yes |
|
||||
| Amazon Drive | Yes | No | Yes | Yes | No | No | No | No | No | No | Yes |
|
||||
| Amazon S3 (or S3 compatible) | No | Yes | No | No | Yes | Yes | Yes | Yes | Yes | No | No |
|
||||
| Backblaze B2 | No | Yes | No | No | Yes | Yes | Yes | Yes | Yes | No | No |
|
||||
| Box | Yes | Yes | Yes | Yes | Yes | No | Yes | No | Yes | Yes | Yes |
|
||||
| Citrix ShareFile | Yes | Yes | Yes | Yes | No | No | No | No | No | No | Yes |
|
||||
| Dropbox | Yes | Yes | Yes | Yes | No | No | Yes | No | Yes | Yes | Yes |
|
||||
| Enterprise File Fabric | Yes | Yes | Yes | Yes | Yes | No | No | No | No | No | Yes |
|
||||
| FTP | No | No | Yes | Yes | No | No | Yes | No | No | No | Yes |
|
||||
| Google Cloud Storage | Yes | Yes | No | No | No | Yes | Yes | No | No | No | No |
|
||||
| Google Drive | Yes | Yes | Yes | Yes | Yes | Yes | Yes | No | Yes | Yes | Yes |
|
||||
| Google Photos | No | No | No | No | No | No | No | No | No | No | No |
|
||||
| HDFS | Yes | No | Yes | Yes | No | No | Yes | No | No | Yes | Yes |
|
||||
| HiDrive | Yes | Yes | Yes | Yes | No | No | Yes | No | No | No | Yes |
|
||||
| HTTP | No | No | No | No | No | No | No | No | No | No | Yes |
|
||||
| Internet Archive | No | Yes | No | No | Yes | Yes | No | No | Yes | Yes | No |
|
||||
| Jottacloud | Yes | Yes | Yes | Yes | Yes | Yes | No | No | Yes | Yes | Yes |
|
||||
| Koofr | Yes | Yes | Yes | Yes | No | No | Yes | No | Yes | Yes | Yes |
|
||||
| Mail.ru Cloud | Yes | Yes | Yes | Yes | Yes | No | No | No | Yes | Yes | Yes |
|
||||
| Mega | Yes | No | Yes | Yes | Yes | No | No | No | Yes | Yes | Yes |
|
||||
| Memory | No | Yes | No | No | No | Yes | Yes | No | No | No | No |
|
||||
| Microsoft Azure Blob Storage | Yes | Yes | No | No | No | Yes | Yes | Yes | No | No | No |
|
||||
| Microsoft OneDrive | Yes | Yes | Yes | Yes | Yes | Yes | No | No | Yes | Yes | Yes |
|
||||
| OpenDrive | Yes | Yes | Yes | Yes | No | No | No | No | No | No | Yes |
|
||||
| OpenStack Swift | Yes ¹ | Yes | No | No | No | Yes | Yes | No | No | Yes | No |
|
||||
| Oracle Object Storage | No | Yes | No | No | Yes | Yes | Yes | Yes | No | No | No |
|
||||
| pCloud | Yes | Yes | Yes | Yes | Yes | No | No | No | Yes | Yes | Yes |
|
||||
| PikPak | Yes | Yes | Yes | Yes | Yes | No | No | No | Yes | Yes | Yes |
|
||||
| premiumize.me | Yes | No | Yes | Yes | No | No | No | No | Yes | Yes | Yes |
|
||||
| put.io | Yes | No | Yes | Yes | Yes | No | Yes | No | No | Yes | Yes |
|
||||
| Proton Drive | Yes | No | Yes | Yes | Yes | No | No | No | No | Yes | Yes |
|
||||
| QingStor | No | Yes | No | No | Yes | Yes | No | No | No | No | No |
|
||||
| Quatrix by Maytech | Yes | Yes | Yes | Yes | No | No | No | No | No | Yes | Yes |
|
||||
| Seafile | Yes | Yes | Yes | Yes | Yes | Yes | Yes | No | Yes | Yes | Yes |
|
||||
| SFTP | No | Yes ⁴| Yes | Yes | No | No | Yes | No | No | Yes | Yes |
|
||||
| Sia | No | No | No | No | No | No | Yes | No | No | No | Yes |
|
||||
| SMB | No | No | Yes | Yes | No | No | Yes | Yes | No | No | Yes |
|
||||
| SugarSync | Yes | Yes | Yes | Yes | No | No | Yes | No | Yes | No | Yes |
|
||||
| Storj | Yes ² | Yes | Yes | No | No | Yes | Yes | No | Yes | No | No |
|
||||
| Uptobox | No | Yes | Yes | Yes | No | No | No | No | No | No | No |
|
||||
| WebDAV | Yes | Yes | Yes | Yes | No | No | Yes ³ | No | No | Yes | Yes |
|
||||
| Yandex Disk | Yes | Yes | Yes | Yes | Yes | No | Yes | No | Yes | Yes | Yes |
|
||||
| Zoho WorkDrive | Yes | Yes | Yes | Yes | No | No | No | No | No | Yes | Yes |
|
||||
| The local filesystem | Yes | No | Yes | Yes | No | No | Yes | Yes | No | Yes | Yes |
|
||||
|
||||
¹ Note Swift implements this in order to delete directory markers but
|
||||
it doesn't actually have a quicker way of deleting files other than
|
||||
deleting them individually.
|
||||
|
||||
² Storj implements this efficiently only for entire buckets. If
|
||||
purging a directory inside a bucket, files are deleted individually.
|
||||
|
||||
³ StreamUpload is not supported with Nextcloud
|
||||
|
||||
⁴ Use the `--sftp-copy-is-hardlink` flag to enable.
|
||||
|
||||
### Purge ###
|
||||
|
||||
This deletes a directory quicker than just deleting all the files in
|
||||
the directory.
|
||||
|
||||
† Note Swift implements this in order to delete directory markers but
|
||||
they don't actually have a quicker way of deleting files other than
|
||||
deleting them individually.
|
||||
|
||||
☨ Storj implements this efficiently only for entire buckets. If
|
||||
purging a directory inside a bucket, files are deleted individually.
|
||||
|
||||
‡ StreamUpload is not supported with Nextcloud
|
||||
|
||||
### Copy ###
|
||||
|
||||
Used when copying an object to and from the same remote. This known
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ start() {
|
|||
echo host=$(docker_ip)
|
||||
echo user=$USER
|
||||
echo pass=$(rclone obscure $PASS)
|
||||
echo copy_is_hardlink=true
|
||||
echo _connect=$(docker_ip):22
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue