drive: Use o.Update and fs.Put to optimise transfers

This commit is contained in:
Nick Craig-Wood 2014-04-18 17:46:57 +01:00
parent 02afcb00e9
commit 8fd59f2e7d
3 changed files with 48 additions and 56 deletions

View File

@ -650,13 +650,46 @@ func (f *FsDrive) ListDir() fs.DirChan {
// Put the object
//
// This assumes that the object doesn't not already exists - if you
// call it when it does exist then it will create a duplicate. Call
// object.Update() in this case.
//
// Copy the reader in to the new object which is returned
//
// The new object may have been created if an error is returned
func (f *FsDrive) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
// Temporary FsObject under construction
fs := &FsObjectDrive{drive: f, remote: remote}
return fs, fs.Update(in, modTime, size)
o := &FsObjectDrive{drive: f, remote: remote}
directory, leaf := splitPath(o.remote)
directoryId, err := f.findDir(directory, true)
if err != nil {
return o, fmt.Errorf("Couldn't find or make directory: %s", err)
}
// Guess the mime type
mimeType := mime.TypeByExtension(path.Ext(o.remote))
if mimeType == "" {
mimeType = "application/octet-stream"
}
modifiedDate := modTime.Format(time.RFC3339Nano)
// Define the metadata for the file we are going to create.
info := &drive.File{
Title: leaf,
Description: leaf,
Parents: []*drive.ParentReference{{Id: directoryId}},
MimeType: mimeType,
ModifiedDate: modifiedDate,
}
// Make the API request to upload metadata and file data.
info, err = f.svc.Files.Insert(info).Media(in).Do()
if err != nil {
return o, fmt.Errorf("Upload failed: %s", err)
}
o.setMetaData(info)
return o, nil
}
// Mkdir creates the container if it doesn't exist
@ -840,58 +873,21 @@ func (o *FsObjectDrive) Open() (in io.ReadCloser, err error) {
return res.Body, nil
}
// Update the object
// Update the already existing object
//
// Copy the reader into the object updating modTime and size
//
// The new object may have been created if an error is returned
func (o *FsObjectDrive) Update(in io.Reader, modTime time.Time, size int64) error {
f := o.drive
directory, leaf := splitPath(o.remote)
directoryId, err := f.findDir(directory, true)
info := &drive.File{
Id: o.id,
ModifiedDate: modTime.Format(time.RFC3339Nano),
}
// Make the API request to upload metadata and file data.
info, err := o.drive.svc.Files.Update(info.Id, info).SetModifiedDate(true).Media(in).Do()
if err != nil {
return fmt.Errorf("Couldn't find or make directory: %s", err)
}
// See if the file already exists
var info *drive.File
found, err := f.listAll(directoryId, leaf, false, true, func(item *drive.File) bool {
info = item
return true
})
if err != nil {
return fmt.Errorf("Error finding file: %s", leaf, err)
}
// Guess the mime type
mimeType := mime.TypeByExtension(path.Ext(o.remote))
if mimeType == "" {
mimeType = "application/octet-stream"
}
modifiedDate := modTime.Format(time.RFC3339Nano)
if found {
// Modify metadata
info.ModifiedDate = modifiedDate
info.MimeType = mimeType
// Make the API request to upload metadata and file data.
info, err = f.svc.Files.Update(info.Id, info).SetModifiedDate(true).Media(in).Do()
} else {
// Define the metadata for the file we are going to create.
info = &drive.File{
Title: leaf,
Description: leaf,
Parents: []*drive.ParentReference{{Id: directoryId}},
MimeType: mimeType,
ModifiedDate: modifiedDate,
}
// Make the API request to upload metadata and file data.
info, err = f.svc.Files.Insert(info).Media(in).Do()
}
if err != nil {
return fmt.Errorf("Upload failed: %s", err)
return fmt.Errorf("Update failed: %s", err)
}
o.setMetaData(info)
return nil

View File

@ -111,9 +111,12 @@ func Copy(f Fs, dst, src Object) {
}
in := NewAccount(in0) // account the transfer
var actionTaken string
if dst != nil {
actionTaken = "Copied (updated existing)"
err = dst.Update(in, src.ModTime(), src.Size())
} else {
actionTaken = "Copied (new)"
dst, err = f.Put(in, src.Remote(), src.ModTime(), src.Size())
}
inErr := in.Close()
@ -133,7 +136,7 @@ func Copy(f Fs, dst, src Object) {
}
return
}
Debug(src, "Copied")
Debug(src, actionTaken)
}
// Check to see if src needs to be copied to dst and if so puts it in out

View File

@ -1,6 +1,4 @@
Todo
* Make a test suite which can run on all the given types of fs
* Copy should use the sync code as it is more efficient at directory listing
* FIXME: ls without an argument for buckets/containers?
* FIXME: More -dry-run checks for object transfer
* Might be quicker to check md5sums first? for swift <-> swift certainly, and maybe for small files
@ -43,10 +41,5 @@ s3
* Otherwise can set metadata
* Returns etag and last modified in bucket list
Drive
* Should keep a note of files we list then call a new method
Object.Update() which would be more efficient than haveing to look
the id up for each file
Bugs
* Non verbose - not sure number transferred got counted up? CHECK