Implement server side copies if possible - fixes #99

Add optional fs.Copier interface

Implemented for
  * swift
  * s3
  * drive
  * dropbox
  * google cloud storage
This commit is contained in:
Nick Craig-Wood 2015-02-14 18:48:08 +00:00
parent fedf81c2b7
commit a96b522958
19 changed files with 355 additions and 40 deletions

View File

@ -66,4 +66,4 @@ retag:
git tag -f $(LAST_TAG)
gen_tests:
cd fstest/fstests && go run gen_tests.go
cd fstest/fstests && go generate

View File

@ -103,6 +103,37 @@ Enter an interactive configuration session.
Prints help on rclone commands and options.
Server Side Copy
----------------
Drive, S3, Dropbox, Swift and Google Cloud Storage support server side
copy.
This means if you want to copy one folder to another then rclone won't
download all the files and re-upload them; it will instruct the server
to copy them in place.
Eg
rclone copy s3:oldbucket s3:newbucket
Will copy the contents of `oldbucket` to `newbucket` without
downloading and re-uploading.
Remotes which don't support server side copy (eg local) **will**
download and re-upload in this case.
Server side copies are used with `sync` and `copy` and will be
identified in the log when using the `-v` flag.
Server side copies will only be attempted if the remote names are the
same.
This can be used when scripting to make aged backups efficiently, eg
rclone sync remote:current-backup remote:previous-backup
rclone sync /path/to/files remote:current-backup
Options
-------

View File

@ -734,6 +734,35 @@ func (f *FsDrive) ListDir() fs.DirChan {
return out
}
// Creates a drive.File info from the parameters passed in and a half
// finished FsObjectDrive which must have setMetaData called on it
//
// Used to create new objects
func (f *FsDrive) createFileInfo(remote string, modTime time.Time, size int64) (*FsObjectDrive, *drive.File, error) {
// Temporary FsObject under construction
o := &FsObjectDrive{
drive: f,
remote: remote,
bytes: size,
}
directory, leaf := splitPath(remote)
directoryId, err := f.findDir(directory, true)
if err != nil {
return nil, nil, fmt.Errorf("Couldn't find or make directory: %s", err)
}
// Define the metadata for the file we are going to create.
createInfo := &drive.File{
Title: leaf,
Description: leaf,
Parents: []*drive.ParentReference{{Id: directoryId}},
MimeType: fs.MimeType(o),
ModifiedDate: modTime.Format(timeFormatOut),
}
return o, createInfo, nil
}
// Put the object
//
// This assumes that the object doesn't not already exists - if you
@ -744,22 +773,9 @@ func (f *FsDrive) ListDir() fs.DirChan {
//
// The new object may have been created if an error is returned
func (f *FsDrive) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
// Temporary FsObject under construction
o := &FsObjectDrive{drive: f, remote: remote}
directory, leaf := splitPath(o.remote)
directoryId, err := f.findDir(directory, true)
o, createInfo, err := f.createFileInfo(remote, modTime, size)
if err != nil {
return o, fmt.Errorf("Couldn't find or make directory: %s", err)
}
// Define the metadata for the file we are going to create.
createInfo := &drive.File{
Title: leaf,
Description: leaf,
Parents: []*drive.ParentReference{{Id: directoryId}},
MimeType: fs.MimeType(o),
ModifiedDate: modTime.Format(timeFormatOut),
return nil, err
}
var info *drive.File
@ -830,6 +846,39 @@ func (fs *FsDrive) Precision() time.Duration {
return time.Millisecond
}
// Copy src to this remote using server side copy operations.
//
// This is stored with the remote path given
//
// It returns the destination Object and a possible error
//
// Will only be called if src.Fs().Name() == f.Name()
//
// If it isn't possible then return fs.ErrorCantCopy
func (f *FsDrive) Copy(src fs.Object, remote string) (fs.Object, error) {
srcObj, ok := src.(*FsObjectDrive)
if !ok {
fs.Debug(src, "Can't copy - not same remote type")
return nil, fs.ErrorCantCopy
}
o, createInfo, err := f.createFileInfo(remote, srcObj.ModTime(), srcObj.bytes)
if err != nil {
return nil, err
}
var info *drive.File
o.drive.call(&err, func() {
info, err = o.drive.svc.Files.Copy(srcObj.id, createInfo).Do()
})
if err != nil {
return nil, err
}
o.setMetaData(info)
return o, nil
}
// Purge deletes all the files and the container
//
// Optional interface: Only implement this if you have a way of
@ -1051,4 +1100,5 @@ func (o *FsObjectDrive) Remove() error {
// Check the interfaces are satisfied
var _ fs.Fs = &FsDrive{}
var _ fs.Purger = &FsDrive{}
var _ fs.Copier = &FsDrive{}
var _ fs.Object = &FsObjectDrive{}

View File

@ -1,7 +1,7 @@
// Test Drive filesystem interface
//
// Automatically generated - DO NOT EDIT
// Regenerate with: go run gen_tests.go or make gen_tests
// Regenerate with: make gen_tests
package drive_test
import (
@ -34,6 +34,7 @@ func TestFsListRoot(t *testing.T) { fstests.TestFsListRoot(t) }
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) }
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }
func TestFsCopy(t *testing.T) { fstests.TestFsCopy(t) }
func TestFsRmdirFull(t *testing.T) { fstests.TestFsRmdirFull(t) }
func TestFsPrecision(t *testing.T) { fstests.TestFsPrecision(t) }
func TestObjectString(t *testing.T) { fstests.TestObjectString(t) }

View File

@ -414,6 +414,35 @@ func (f *FsDropbox) Precision() time.Duration {
return fs.ModTimeNotSupported
}
// Copy src to this remote using server side copy operations.
//
// This is stored with the remote path given
//
// It returns the destination Object and a possible error
//
// Will only be called if src.Fs().Name() == f.Name()
//
// If it isn't possible then return fs.ErrorCantCopy
func (f *FsDropbox) Copy(src fs.Object, remote string) (fs.Object, error) {
srcObj, ok := src.(*FsObjectDropbox)
if !ok {
fs.Debug(src, "Can't copy - not same remote type")
return nil, fs.ErrorCantCopy
}
// Temporary FsObject under construction
dstObj := &FsObjectDropbox{dropbox: f, remote: remote}
srcPath := srcObj.remotePath()
dstPath := dstObj.remotePath()
entry, err := f.db.Copy(srcPath, dstPath, false)
if err != nil {
return nil, fmt.Errorf("Copy failed: %s", err)
}
dstObj.setMetadataFromEntry(entry)
return dstObj, nil
}
// Purge deletes all the files and the container
//
// Optional interface: Only implement this if you have a way of
@ -575,5 +604,6 @@ func (o *FsObjectDropbox) Remove() error {
// Check the interfaces are satisfied
var _ fs.Fs = &FsDropbox{}
var _ fs.Copier = &FsDropbox{}
var _ fs.Purger = &FsDropbox{}
var _ fs.Object = &FsObjectDropbox{}

View File

@ -1,7 +1,7 @@
// Test Dropbox filesystem interface
//
// Automatically generated - DO NOT EDIT
// Regenerate with: go run gen_tests.go or make gen_tests
// Regenerate with: make gen_tests
package dropbox_test
import (
@ -34,6 +34,7 @@ func TestFsListRoot(t *testing.T) { fstests.TestFsListRoot(t) }
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) }
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }
func TestFsCopy(t *testing.T) { fstests.TestFsCopy(t) }
func TestFsRmdirFull(t *testing.T) { fstests.TestFsRmdirFull(t) }
func TestFsPrecision(t *testing.T) { fstests.TestFsPrecision(t) }
func TestObjectString(t *testing.T) { fstests.TestObjectString(t) }

View File

@ -24,6 +24,7 @@ var (
fsRegistry []*FsInfo
// Error returned by NewFs if not found in config file
NotFoundInConfigFile = fmt.Errorf("Didn't find section in config file")
ErrorCantCopy = fmt.Errorf("Can't copy object - incompatible remotes")
)
// Filesystem info
@ -149,6 +150,19 @@ type Purger interface {
Purge() error
}
type Copier interface {
// Copy src to this remote using server side copy operations.
//
// This is stored with the remote path given
//
// It returns the destination Object and a possible error
//
// Will only be called if src.Fs().Name() == f.Name()
//
// If it isn't possible then return fs.ErrorCantCopy
Copy(src Object, remote string) (Object, error)
}
// An optional interface for error as to whether the operation should be retried
//
// This should be returned from Update or Put methods as required

View File

@ -89,5 +89,23 @@ func (f *Limited) Precision() time.Duration {
return f.fs.Precision()
}
// Copy src to this remote using server side copy operations.
//
// This is stored with the remote path given
//
// It returns the destination Object and a possible error
//
// Will only be called if src.Fs().Name() == f.Name()
//
// If it isn't possible then return fs.ErrorCantCopy
func (f *Limited) Copy(src Object, remote string) (Object, error) {
fCopy, ok := f.fs.(Copier)
if !ok {
return nil, ErrorCantCopy
}
return fCopy.Copy(src, remote)
}
// Check the interfaces are satisfied
var _ Fs = &Limited{}
var _ Copier = &Limited{}

View File

@ -173,24 +173,40 @@ func Copy(f Fs, dst, src Object) {
const maxTries = 10
tries := 0
doUpdate := dst != nil
var err, inErr error
tryAgain:
in0, err := src.Open()
if err != nil {
Stats.Error()
ErrorLog(src, "Failed to open: %s", err)
return
}
in := NewAccount(in0) // account the transfer
var actionTaken string
if doUpdate {
actionTaken = "Copied (updated existing)"
err = dst.Update(in, src.ModTime(), src.Size())
// Try server side copy first - if has optional interface and
// is same underlying remote
actionTaken := "Copied (server side copy)"
if fCopy, ok := f.(Copier); ok && src.Fs().Name() == f.Name() {
var newDst Object
newDst, err = fCopy.Copy(src, src.Remote())
if err == nil {
dst = newDst
}
} else {
actionTaken = "Copied (new)"
dst, err = f.Put(in, src.Remote(), src.ModTime(), src.Size())
err = ErrorCantCopy
}
// If can't server side copy, do it manually
if err == ErrorCantCopy {
var in0 io.ReadCloser
in0, err = src.Open()
if err != nil {
Stats.Error()
ErrorLog(src, "Failed to open: %s", err)
return
}
in := NewAccount(in0) // account the transfer
if doUpdate {
actionTaken = "Copied (updated existing)"
err = dst.Update(in, src.ModTime(), src.Size())
} else {
actionTaken = "Copied (new)"
dst, err = f.Put(in, src.Remote(), src.ModTime(), src.Size())
}
inErr = in.Close()
}
inErr := in.Close()
// Retry if err returned a retry error
if r, ok := err.(Retry); ok && r.Retry() && tries < maxTries {
tries++
@ -279,7 +295,7 @@ func PairChecker(in ObjectPairChan, out ObjectPairChan, wg *sync.WaitGroup) {
}
// Read Objects on in and copy them
func Copier(in ObjectPairChan, fdst Fs, wg *sync.WaitGroup) {
func PairCopier(in ObjectPairChan, fdst Fs, wg *sync.WaitGroup) {
defer wg.Done()
for pair := range in {
src := pair.src
@ -364,7 +380,7 @@ func Sync(fdst, fsrc Fs, Delete bool) error {
var copierWg sync.WaitGroup
copierWg.Add(Config.Transfers)
for i := 0; i < Config.Transfers; i++ {
go Copier(to_be_uploaded, fdst, &copierWg)
go PairCopier(to_be_uploaded, fdst, &copierWg)
}
go func() {

View File

@ -127,6 +127,28 @@ func TestCopy(t *testing.T) {
fstest.CheckListingWithPrecision(t, fremote, items, fs.Config.ModifyWindow)
}
// Test a server side copy if possible, or the backup path if not
func TestServerSideCopy(t *testing.T) {
fremoteCopy, finaliseCopy, err := fstest.RandomRemote(*RemoteName, *SubDir)
if err != nil {
t.Fatalf("Failed to open remote copy %q: %v", *RemoteName, err)
}
defer finaliseCopy()
t.Logf("Server side copy (if possible) %v -> %v", fremote, fremoteCopy)
err = fs.Sync(fremoteCopy, fremote, false)
if err != nil {
t.Fatalf("Server Side Copy failed: %v", err)
}
items := []fstest.Item{
{Path: "sub dir/hello world", Size: 11, ModTime: t1, Md5sum: "5eb63bbbe01eeed093cb22bb8f5acdc3"},
}
fstest.CheckListingWithPrecision(t, fremote, items, fs.Config.ModifyWindow)
fstest.CheckListingWithPrecision(t, fremoteCopy, items, fs.Config.ModifyWindow)
}
func TestLsd(t *testing.T) {
var buf bytes.Buffer
err := fs.ListDir(fremote, &buf)

View File

@ -1,4 +1,8 @@
// Generic tests for testing the Fs and Object interfaces
//
// Run go generate to write the tests for the remotes
//go:generate go run gen_tests.go
package fstests
import (
@ -247,6 +251,41 @@ func TestFsListFile1and2(t *testing.T) {
fstest.CheckListing(t, remote, []fstest.Item{file1, file2})
}
func TestFsCopy(t *testing.T) {
skipIfNotOk(t)
// Check have Copy
_, ok := remote.(fs.Copier)
if !ok {
t.Skip("FS has no Copier interface")
}
var file1Copy = file1
file1Copy.Path += "-copy"
// do the copy
src := findObject(t, file1.Path)
dst, err := remote.(fs.Copier).Copy(src, file1Copy.Path)
if err != nil {
t.Errorf("Copy failed: %v", err)
}
// check file exists in new listing
fstest.CheckListing(t, remote, []fstest.Item{file1, file2, file1Copy})
// Check dst lightly - list above has checked ModTime/Md5sum
if dst.Remote() != file1Copy.Path {
t.Errorf("object path: want %q got %q", file1Copy.Path, dst.Remote())
}
// Delete copy
err = dst.Remove()
if err != nil {
t.Fatal("Remove copy error", err)
}
}
func TestFsRmdirFull(t *testing.T) {
skipIfNotOk(t)
err := remote.Rmdir()

View File

@ -92,7 +92,7 @@ func generateTestProgram(t *template.Template, fns []string, Fsname string) {
}
data := Data{
Regenerate: "go run gen_tests.go or make gen_tests",
Regenerate: "make gen_tests",
FsName: fsname,
UpperFsName: Fsname,
TestName: TestName,

View File

@ -401,6 +401,38 @@ func (fs *FsStorage) Precision() time.Duration {
return time.Nanosecond
}
// Copy src to this remote using server side copy operations.
//
// This is stored with the remote path given
//
// It returns the destination Object and a possible error
//
// Will only be called if src.Fs().Name() == f.Name()
//
// If it isn't possible then return fs.ErrorCantCopy
func (f *FsStorage) Copy(src fs.Object, remote string) (fs.Object, error) {
srcObj, ok := src.(*FsObjectStorage)
if !ok {
fs.Debug(src, "Can't copy - not same remote type")
return nil, fs.ErrorCantCopy
}
// Temporary FsObject under construction
dstObj := &FsObjectStorage{storage: f, remote: remote}
srcBucket := srcObj.storage.bucket
srcObject := srcObj.storage.root + srcObj.remote
dstBucket := f.bucket
dstObject := f.root + remote
newObject, err := f.svc.Objects.Copy(srcBucket, srcObject, dstBucket, dstObject, nil).Do()
if err != nil {
return nil, err
}
// Set the metadata for the new object while we have it
dstObj.setMetaData(newObject)
return dstObj, nil
}
// ------------------------------------------------------------
// Return the parent Fs
@ -578,4 +610,5 @@ func (o *FsObjectStorage) Remove() error {
// Check the interfaces are satisfied
var _ fs.Fs = &FsStorage{}
var _ fs.Copier = &FsStorage{}
var _ fs.Object = &FsObjectStorage{}

View File

@ -1,7 +1,7 @@
// Test GoogleCloudStorage filesystem interface
//
// Automatically generated - DO NOT EDIT
// Regenerate with: go run gen_tests.go or make gen_tests
// Regenerate with: make gen_tests
package googlecloudstorage_test
import (
@ -34,6 +34,7 @@ func TestFsListRoot(t *testing.T) { fstests.TestFsListRoot(t) }
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) }
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }
func TestFsCopy(t *testing.T) { fstests.TestFsCopy(t) }
func TestFsRmdirFull(t *testing.T) { fstests.TestFsRmdirFull(t) }
func TestFsPrecision(t *testing.T) { fstests.TestFsPrecision(t) }
func TestObjectString(t *testing.T) { fstests.TestObjectString(t) }

View File

@ -1,7 +1,7 @@
// Test Local filesystem interface
//
// Automatically generated - DO NOT EDIT
// Regenerate with: go run gen_tests.go or make gen_tests
// Regenerate with: make gen_tests
package local_test
import (
@ -34,6 +34,7 @@ func TestFsListRoot(t *testing.T) { fstests.TestFsListRoot(t) }
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) }
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }
func TestFsCopy(t *testing.T) { fstests.TestFsCopy(t) }
func TestFsRmdirFull(t *testing.T) { fstests.TestFsRmdirFull(t) }
func TestFsPrecision(t *testing.T) { fstests.TestFsPrecision(t) }
func TestObjectString(t *testing.T) { fstests.TestObjectString(t) }

View File

@ -482,6 +482,37 @@ func (f *FsS3) Precision() time.Duration {
return time.Nanosecond
}
// Copy src to this remote using server side copy operations.
//
// This is stored with the remote path given
//
// It returns the destination Object and a possible error
//
// Will only be called if src.Fs().Name() == f.Name()
//
// If it isn't possible then return fs.ErrorCantCopy
func (f *FsS3) Copy(src fs.Object, remote string) (fs.Object, error) {
srcObj, ok := src.(*FsObjectS3)
if !ok {
fs.Debug(src, "Can't copy - not same remote type")
return nil, fs.ErrorCantCopy
}
srcFs := srcObj.s3
key := f.root + remote
source := srcFs.bucket + "/" + srcFs.root + srcObj.remote
req := s3.CopyObjectInput{
Bucket: &f.bucket,
Key: &key,
CopySource: &source,
MetadataDirective: aws.String(s3.MetadataDirectiveCopy),
}
_, err := f.c.CopyObject(&req)
if err != nil {
return nil, err
}
return f.NewFsObject(remote), err
}
// ------------------------------------------------------------
// Return the parent Fs
@ -679,4 +710,5 @@ func (o *FsObjectS3) Remove() error {
// Check the interfaces are satisfied
var _ fs.Fs = &FsS3{}
var _ fs.Copier = &FsS3{}
var _ fs.Object = &FsObjectS3{}

View File

@ -1,7 +1,7 @@
// Test S3 filesystem interface
//
// Automatically generated - DO NOT EDIT
// Regenerate with: go run gen_tests.go or make gen_tests
// Regenerate with: make gen_tests
package s3_test
import (
@ -34,6 +34,7 @@ func TestFsListRoot(t *testing.T) { fstests.TestFsListRoot(t) }
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) }
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }
func TestFsCopy(t *testing.T) { fstests.TestFsCopy(t) }
func TestFsRmdirFull(t *testing.T) { fstests.TestFsRmdirFull(t) }
func TestFsPrecision(t *testing.T) { fstests.TestFsPrecision(t) }
func TestObjectString(t *testing.T) { fstests.TestObjectString(t) }

View File

@ -328,6 +328,29 @@ func (fs *FsSwift) Precision() time.Duration {
return time.Nanosecond
}
// Copy src to this remote using server side copy operations.
//
// This is stored with the remote path given
//
// It returns the destination Object and a possible error
//
// Will only be called if src.Fs().Name() == f.Name()
//
// If it isn't possible then return fs.ErrorCantCopy
func (f *FsSwift) Copy(src fs.Object, remote string) (fs.Object, error) {
srcObj, ok := src.(*FsObjectSwift)
if !ok {
fs.Debug(src, "Can't copy - not same remote type")
return nil, fs.ErrorCantCopy
}
srcFs := srcObj.swift
_, err := f.c.ObjectCopy(srcFs.container, srcFs.root+srcObj.remote, f.container, f.root+remote, nil)
if err != nil {
return nil, err
}
return f.NewFsObject(remote), nil
}
// ------------------------------------------------------------
// Return the parent Fs
@ -446,4 +469,5 @@ func (o *FsObjectSwift) Remove() error {
// Check the interfaces are satisfied
var _ fs.Fs = &FsSwift{}
var _ fs.Copier = &FsSwift{}
var _ fs.Object = &FsObjectSwift{}

View File

@ -1,7 +1,7 @@
// Test Swift filesystem interface
//
// Automatically generated - DO NOT EDIT
// Regenerate with: go run gen_tests.go or make gen_tests
// Regenerate with: make gen_tests
package swift_test
import (
@ -34,6 +34,7 @@ func TestFsListRoot(t *testing.T) { fstests.TestFsListRoot(t) }
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) }
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }
func TestFsCopy(t *testing.T) { fstests.TestFsCopy(t) }
func TestFsRmdirFull(t *testing.T) { fstests.TestFsRmdirFull(t) }
func TestFsPrecision(t *testing.T) { fstests.TestFsPrecision(t) }
func TestObjectString(t *testing.T) { fstests.TestObjectString(t) }