community
/
rclone
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Factor Fs.Put into Object.Update and call Update rather than Put if possible

This commit is contained in:
Nick Craig-Wood 2014-04-18 17:04:21 +01:00
parent d6a5bfe2d4
commit 02afcb00e9
6 changed files with 135 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

111
drive/drive.go
View File

@ -648,63 +648,15 @@ func (f *FsDrive) ListDir() fs.DirChan {
return out
}
// Put the FsObject into the container
// Put the object
//
// Copy the reader in to the new object which is returned
//
// The new object may have been created
// The new object may have been created if an error is returned
func (f *FsDrive) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
// Temporary FsObject under construction
fs := &FsObjectDrive{drive: f, remote: remote}
directory, leaf := splitPath(remote)
directoryId, err := f.findDir(directory, true)
if err != nil {
return nil, fmt.Errorf("Couldn't find or make directory: %s", err)
}
// See if the file already exists
var info *drive.File
found, err := f.listAll(directoryId, leaf, false, true, func(item *drive.File) bool {
info = item
return true
})
if err != nil {
return nil, fmt.Errorf("Error finding file: %s", leaf, err)
}
// Guess the mime type
mimeType := mime.TypeByExtension(path.Ext(remote))
if mimeType == "" {
mimeType = "application/octet-stream"
}
modifiedDate := modTime.Format(time.RFC3339Nano)
if found {
// Modify metadata
info.ModifiedDate = modifiedDate
info.MimeType = mimeType
// Make the API request to upload metadata and file data.
info, err = f.svc.Files.Update(info.Id, info).SetModifiedDate(true).Media(in).Do()
} else {
// Define the metadata for the file we are going to create.
info = &drive.File{
Title: leaf,
Description: leaf,
Parents: []*drive.ParentReference{{Id: directoryId}},
MimeType: mimeType,
ModifiedDate: modifiedDate,
}
// Make the API request to upload metadata and file data.
info, err = f.svc.Files.Insert(info).Media(in).Do()
}
if err != nil {
return nil, fmt.Errorf("Upload failed: %s", err)
}
fs.setMetaData(info)
return fs, nil
return fs, fs.Update(in, modTime, size)
}
// Mkdir creates the container if it doesn't exist
@ -888,6 +840,63 @@ func (o *FsObjectDrive) Open() (in io.ReadCloser, err error) {
return res.Body, nil
}
// Update the object
//
// Copy the reader into the object updating modTime and size
//
// The new object may have been created if an error is returned
func (o *FsObjectDrive) Update(in io.Reader, modTime time.Time, size int64) error {
f := o.drive
directory, leaf := splitPath(o.remote)
directoryId, err := f.findDir(directory, true)
if err != nil {
return fmt.Errorf("Couldn't find or make directory: %s", err)
}
// See if the file already exists
var info *drive.File
found, err := f.listAll(directoryId, leaf, false, true, func(item *drive.File) bool {
info = item
return true
})
if err != nil {
return fmt.Errorf("Error finding file: %s", leaf, err)
}
// Guess the mime type
mimeType := mime.TypeByExtension(path.Ext(o.remote))
if mimeType == "" {
mimeType = "application/octet-stream"
}
modifiedDate := modTime.Format(time.RFC3339Nano)
if found {
// Modify metadata
info.ModifiedDate = modifiedDate
info.MimeType = mimeType
// Make the API request to upload metadata and file data.
info, err = f.svc.Files.Update(info.Id, info).SetModifiedDate(true).Media(in).Do()
} else {
// Define the metadata for the file we are going to create.
info = &drive.File{
Title: leaf,
Description: leaf,
Parents: []*drive.ParentReference{{Id: directoryId}},
MimeType: mimeType,
ModifiedDate: modifiedDate,
}
// Make the API request to upload metadata and file data.
info, err = f.svc.Files.Insert(info).Media(in).Do()
}
if err != nil {
return fmt.Errorf("Upload failed: %s", err)
}
o.setMetaData(info)
return nil
}
// Remove an object
func (o *FsObjectDrive) Remove() error {
return o.drive.svc.Files.Delete(o.id).Do()

3
fs/fs.go
View File

@ -103,6 +103,9 @@ type Object interface {
// Open opens the file for read. Call Close() on the returned io.ReadCloser
Open() (io.ReadCloser, error)
// Update in to the object with the modTime given of the given size
Update(in io.Reader, modTime time.Time, size int64) error
// Storable says whether this object can be stored
Storable() bool

16
fs/operations.go
View File

@ -97,8 +97,12 @@ func Equal(src, dst Object) bool {
return true
}
// Copy src object to f
func Copy(f Fs, src Object) {
// Copy src object to dst or f if nil
//
// If dst is nil then the object must not exist already. If you do
// call Copy() with dst nil on a pre-existing file then some filing
// systems (eg Drive) may duplicate the file.
func Copy(f Fs, dst, src Object) {
in0, err := src.Open()
if err != nil {
Stats.Error()
@ -107,7 +111,11 @@ func Copy(f Fs, src Object) {
}
in := NewAccount(in0) // account the transfer
dst, err := f.Put(in, src.Remote(), src.ModTime(), src.Size())
if dst != nil {
err = dst.Update(in, src.ModTime(), src.Size())
} else {
dst, err = f.Put(in, src.Remote(), src.ModTime(), src.Size())
}
inErr := in.Close()
if err == nil {
err = inErr
@ -167,7 +175,7 @@ func Copier(in ObjectPairChan, fdst Fs, wg *sync.WaitGroup) {
for pair := range in {
src := pair.src
Stats.Transferring(src)
Copy(fdst, src)
Copy(fdst, pair.dst, src)
Stats.DoneTransferring(src)
}
}

52
local/local.go
View File

@ -164,30 +164,7 @@ func (f *FsLocal) Put(in io.Reader, remote string, modTime time.Time, size int64
dstPath := filepath.Join(f.root, remote)
// Temporary FsObject under construction
fs := &FsObjectLocal{local: f, remote: remote, path: dstPath}
dir := path.Dir(dstPath)
err := os.MkdirAll(dir, 0770)
if err != nil {
return fs, err
}
out, err := os.Create(dstPath)
if err != nil {
return fs, err
}
_, err = io.Copy(out, in)
outErr := out.Close()
if err != nil {
return fs, err
}
if outErr != nil {
return fs, outErr
}
// Set the mtime
fs.SetModTime(modTime)
return fs, err
return fs, fs.Update(in, modTime, size)
}
// Mkdir creates the directory if it doesn't exist
@ -335,6 +312,33 @@ func (o *FsObjectLocal) Open() (in io.ReadCloser, err error) {
return
}
// Update the object from in with modTime and size
func (o *FsObjectLocal) Update(in io.Reader, modTime time.Time, size int64) error {
dir := path.Dir(o.path)
err := os.MkdirAll(dir, 0770)
if err != nil {
return err
}
out, err := os.Create(o.path)
if err != nil {
return err
}
_, err = io.Copy(out, in)
outErr := out.Close()
if err != nil {
return err
}
if outErr != nil {
return outErr
}
// Set the mtime
o.SetModTime(modTime)
return nil
}
// Stat a FsObject into info
func (o *FsObjectLocal) lstat() error {
info, err := os.Lstat(o.path)

32
s3/s3.go
View File

@ -290,20 +290,7 @@ func (f *FsS3) ListDir() fs.DirChan {
func (f *FsS3) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
// Temporary FsObject under construction
fs := &FsObjectS3{s3: f, remote: remote}
// Set the mtime in the headers
headers := s3.Headers{
metaMtime: swift.TimeToFloatString(modTime),
}
// Guess the content type
contentType := mime.TypeByExtension(path.Ext(remote))
if contentType == "" {
contentType = "application/octet-stream"
}
_, err := fs.s3.b.PutReaderHeaders(remote, in, size, contentType, f.perm, headers)
return fs, err
return fs, fs.Update(in, modTime, size)
}
// Mkdir creates the bucket if it doesn't exist
@ -438,6 +425,23 @@ func (o *FsObjectS3) Open() (in io.ReadCloser, err error) {
return
}
// Update the Object from in with modTime and size
func (o *FsObjectS3) Update(in io.Reader, modTime time.Time, size int64) error {
// Set the mtime in the headers
headers := s3.Headers{
metaMtime: swift.TimeToFloatString(modTime),
}
// Guess the content type
contentType := mime.TypeByExtension(path.Ext(o.remote))
if contentType == "" {
contentType = "application/octet-stream"
}
_, err := o.s3.b.PutReaderHeaders(o.remote, in, size, contentType, o.s3.perm, headers)
return err
}
// Remove an object
func (o *FsObjectS3) Remove() error {
return o.s3.b.Del(o.remote)

22
swift/swift.go
View File

@ -210,20 +210,15 @@ func (f *FsSwift) ListDir() fs.DirChan {
return out
}
// Put the FsObject into the container
// Put the object into the container
//
// Copy the reader in to the new object which is returned
//
// The new object may have been created
// The new object may have been created if an error is returned
func (f *FsSwift) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
// Temporary FsObject under construction
fs := &FsObjectSwift{swift: f, remote: remote}
// Set the mtime
m := swift.Metadata{}
m.SetModTime(modTime)
_, err := f.c.ObjectPut(f.container, remote, in, true, "", "", m.ObjectHeaders())
return fs, err
return fs, fs.Update(in, modTime, size)
}
// Mkdir creates the container if it doesn't exist
@ -337,6 +332,17 @@ func (o *FsObjectSwift) Open() (in io.ReadCloser, err error) {
return
}
// Update the object with the contents of the io.Reader, modTime and size
//
// The new object may have been created if an error is returned
func (o *FsObjectSwift) Update(in io.Reader, modTime time.Time, size int64) error {
// Set the mtime
m := swift.Metadata{}
m.SetModTime(modTime)
_, err := o.swift.c.ObjectPut(o.swift.container, o.remote, in, true, "", "", m.ObjectHeaders())
return err
}
// Remove an object
func (o *FsObjectSwift) Remove() error {
return o.swift.c.ObjectDelete(o.swift.container, o.remote)