From ef06371c93e00abc69a64f76ed4b3267378e875b Mon Sep 17 00:00:00 2001 From: klauspost Date: Thu, 18 Feb 2016 12:35:25 +0100 Subject: [PATCH] Create separate interface for object information. Take out read-only information about a Fs in a separate struct to limit access. See discussion at #282. --- amazonclouddrive/amazonclouddrive.go | 13 ++- b2/b2.go | 15 +-- drive/drive.go | 14 ++- dropbox/dropbox.go | 12 +-- fs/fs.go | 116 +++++++++++++++++------ fs/limited.go | 5 +- fs/operations.go | 4 +- fs/operations_test.go | 3 +- fstest/fstests/fstests.go | 6 +- googlecloudstorage/googlecloudstorage.go | 15 +-- hubic/hubic.go | 2 +- local/local.go | 15 +-- onedrive/onedrive.go | 17 +++- s3/s3.go | 14 +-- swift/swift.go | 15 +-- yandex/yandex.go | 19 ++-- 16 files changed, 190 insertions(+), 95 deletions(-) diff --git a/amazonclouddrive/amazonclouddrive.go b/amazonclouddrive/amazonclouddrive.go index f1853c130..5a5869748 100644 --- a/amazonclouddrive/amazonclouddrive.go +++ b/amazonclouddrive/amazonclouddrive.go @@ -62,7 +62,7 @@ var ( // Register with Fs func init() { - fs.Register(&fs.Info{ + fs.Register(&fs.RegInfo{ Name: "amazon cloud drive", NewFs: NewFs, Config: func(name string) { @@ -521,7 +521,9 @@ func (f *Fs) ListDir() fs.DirChan { // Copy the reader in to the new object which is returned // // The new object may have been created if an error is returned -func (f *Fs) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) { +func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) { + remote := src.Remote() + size := src.Size() // Temporary Object under construction o := &Object{ fs: f, @@ -538,7 +540,7 @@ func (f *Fs) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs var info *acd.File var resp *http.Response err = f.pacer.CallNoRetry(func() (bool, error) { - if size != 0 { + if src.Size() != 0 { info, resp, err = folder.Put(in, leaf) } else { info, resp, err = folder.PutSized(in, size, leaf) @@ -663,7 +665,7 @@ func (f *Fs) Purge() error { // ------------------------------------------------------------ // Fs returns the parent Fs -func (o *Object) Fs() fs.Fs { +func (o *Object) Fs() fs.Info { return o.fs } @@ -774,7 +776,8 @@ func (o *Object) Open() (in io.ReadCloser, err error) { // Update the object with the contents of the io.Reader, modTime and size // // The new object may have been created if an error is returned -func (o *Object) Update(in io.Reader, modTime time.Time, size int64) error { +func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error { + size := src.Size() file := acd.File{Node: o.info} var info *acd.File var resp *http.Response diff --git a/b2/b2.go b/b2/b2.go index 3b2f5cecf..335acef42 100644 --- a/b2/b2.go +++ b/b2/b2.go @@ -39,7 +39,7 @@ const ( // Register with Fs func init() { - fs.Register(&fs.Info{ + fs.Register(&fs.RegInfo{ Name: "b2", NewFs: NewFs, Options: []fs.Option{{ @@ -462,13 +462,13 @@ func (f *Fs) ListDir() fs.DirChan { // Copy the reader in to the new object which is returned // // The new object may have been created if an error is returned -func (f *Fs) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) { +func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) { // Temporary Object under construction fs := &Object{ fs: f, - remote: remote, + remote: src.Remote(), } - return fs, fs.Update(in, modTime, size) + return fs, fs.Update(in, src) } // Mkdir creates the bucket if it doesn't exist @@ -599,7 +599,7 @@ func (f *Fs) Hashes() fs.HashSet { // ------------------------------------------------------------ // Fs returns the parent Fs -func (o *Object) Fs() fs.Fs { +func (o *Object) Fs() fs.Info { return o.fs } @@ -875,7 +875,10 @@ func urlEncode(in string) string { // Update the object with the contents of the io.Reader, modTime and size // // The new object may have been created if an error is returned -func (o *Object) Update(in io.Reader, modTime time.Time, size int64) (err error) { +func (o *Object) Update(in io.Reader, src fs.ObjectInfo) (err error) { + size := src.Size() + modTime := src.ModTime() + // Open a temp file to copy the input fd, err := ioutil.TempFile("", "rclone-b2-") if err != nil { diff --git a/drive/drive.go b/drive/drive.go index 7fb67f90c..d541420e2 100644 --- a/drive/drive.go +++ b/drive/drive.go @@ -84,7 +84,7 @@ var ( // Register with Fs func init() { - fs.Register(&fs.Info{ + fs.Register(&fs.RegInfo{ Name: "drive", NewFs: NewFs, Config: func(name string) { @@ -600,7 +600,11 @@ func (f *Fs) createFileInfo(remote string, modTime time.Time, size int64) (*Obje // Copy the reader in to the new object which is returned // // The new object may have been created if an error is returned -func (f *Fs) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) { +func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) { + remote := src.Remote() + size := src.Size() + modTime := src.ModTime() + o, createInfo, err := f.createFileInfo(remote, modTime, size) if err != nil { return nil, err @@ -818,7 +822,7 @@ func (f *Fs) Hashes() fs.HashSet { // ------------------------------------------------------------ // Fs returns the parent Fs -func (o *Object) Fs() fs.Fs { +func (o *Object) Fs() fs.Info { return o.fs } @@ -1025,7 +1029,9 @@ func (o *Object) Open() (in io.ReadCloser, err error) { // Copy the reader into the object updating modTime and size // // The new object may have been created if an error is returned -func (o *Object) Update(in io.Reader, modTime time.Time, size int64) error { +func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error { + size := src.Size() + modTime := src.ModTime() if o.isDocument { return fmt.Errorf("Can't update a google document") } diff --git a/dropbox/dropbox.go b/dropbox/dropbox.go index 9332550c6..73aa8a1e9 100644 --- a/dropbox/dropbox.go +++ b/dropbox/dropbox.go @@ -44,7 +44,7 @@ var ( // Register with Fs func init() { - fs.Register(&fs.Info{ + fs.Register(&fs.RegInfo{ Name: "dropbox", NewFs: NewFs, Config: configHelper, @@ -379,13 +379,13 @@ func (rc *readCloser) Close() error { // Copy the reader in to the new object which is returned // // The new object may have been created if an error is returned -func (f *Fs) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) { +func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) { // Temporary Object under construction o := &Object{ fs: f, - remote: remote, + remote: src.Remote(), } - return o, o.Update(in, modTime, size) + return o, o.Update(in, src) } // Mkdir creates the container if it doesn't exist @@ -531,7 +531,7 @@ func (f *Fs) Hashes() fs.HashSet { // ------------------------------------------------------------ // Fs returns the parent Fs -func (o *Object) Fs() fs.Fs { +func (o *Object) Fs() fs.Info { return o.fs } @@ -656,7 +656,7 @@ func (o *Object) Open() (in io.ReadCloser, err error) { // Copy the reader into the object updating modTime and size // // The new object may have been created if an error is returned -func (o *Object) Update(in io.Reader, modTime time.Time, size int64) error { +func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error { remote := o.remotePath() if ignoredFiles.MatchString(remote) { fs.Log(o, "File name disallowed - not uploading") diff --git a/fs/fs.go b/fs/fs.go index 235d93155..18140a376 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -22,7 +22,7 @@ const ( // Globals var ( // Filesystem registry - fsRegistry []*Info + fsRegistry []*RegInfo // ErrorNotFoundInConfigFile is returned by NewFs if not found in config file ErrorNotFoundInConfigFile = fmt.Errorf("Didn't find section in config file") ErrorCantPurge = fmt.Errorf("Can't purge directory") @@ -32,8 +32,8 @@ var ( ErrorDirExists = fmt.Errorf("Can't copy directory - destination already exists") ) -// Info information about a filesystem -type Info struct { +// RegInfo provides information about a filesystem +type RegInfo struct { // Name of this fs Name string // Create a new file system. If root refers to an existing @@ -63,20 +63,13 @@ type OptionExample struct { // Register a filesystem // // Fs modules should use this in an init() function -func Register(info *Info) { +func Register(info *RegInfo) { fsRegistry = append(fsRegistry, info) } // Fs is the interface a cloud storage system must provide type Fs interface { - // Name of the remote (as passed into NewFs) - Name() string - - // Root of the remote (as passed into NewFs) - Root() string - - // String returns a description of the FS - String() string + Info // List the Fs into a channel List() ObjectsChan @@ -92,7 +85,7 @@ type Fs interface { // May create the object even if it returns an error - if so // will return the object and the error, otherwise will return // nil and the error - Put(in io.Reader, remote string, modTime time.Time, size int64) (Object, error) + Put(in io.Reader, src ObjectInfo) (Object, error) // Mkdir makes the directory (container, bucket) // @@ -103,6 +96,18 @@ type Fs interface { // // Return an error if it doesn't exist or isn't empty Rmdir() error +} + +// Info provides an interface to reading information about a filesystem. +type Info interface { + // Name of the remote (as passed into NewFs) + Name() string + + // Root of the remote (as passed into NewFs) + Root() string + + // String returns a description of the FS + String() string // Precision of the ModTimes in this Fs Precision() time.Duration @@ -113,40 +118,45 @@ type Fs interface { // Object is a filesystem like object provided by an Fs type Object interface { + ObjectInfo + // String returns a description of the Object String() string - // Fs returns the Fs that this object is part of - Fs() Fs + // SetModTime sets the metadata on the object to set the modification date + SetModTime(time.Time) + + // Open opens the file for read. Call Close() on the returned io.ReadCloser + Open() (io.ReadCloser, error) + + // Update in to the object with the modTime given of the given size + Update(in io.Reader, src ObjectInfo) error + + // Removes this object + Remove() error +} + +// ObjectInfo contains information about an object. +type ObjectInfo interface { + // Fs returns read only access to the Fs that this object is part of + Fs() Info // Remote returns the remote path Remote() string - // Md5sum returns the md5 checksum of the file - // If no Md5sum is available it returns "" + // Hash returns the selected checksum of the file + // If no checksum is available it returns "" Hash(HashType) (string, error) // ModTime returns the modification date of the file // It should return a best guess if one isn't available ModTime() time.Time - // SetModTime sets the metadata on the object to set the modification date - SetModTime(time.Time) - // Size returns the size of the file Size() int64 - // Open opens the file for read. Call Close() on the returned io.ReadCloser - Open() (io.ReadCloser, error) - - // Update in to the object with the modTime given of the given size - Update(in io.Reader, modTime time.Time, size int64) error - // Storable says whether this object can be stored Storable() bool - - // Removes this object - Remove() error } // Purger is an optional interfaces for Fs @@ -236,7 +246,7 @@ type DirChan chan *Dir // Find looks for an Info object for the name passed in // // Services are looked up in the config file -func Find(name string) (*Info, error) { +func Find(name string) (*RegInfo, error) { for _, item := range fsRegistry { if item.Name == name { return item, nil @@ -316,3 +326,49 @@ func CheckClose(c io.Closer, err *error) { *err = cerr } } + +// NewStaticObjectInfo returns a static ObjectInfo +// If hashes is nil and fs is not nil, the hash map will be replaced with +// empty hashes of the types supported by the fs. +func NewStaticObjectInfo(remote string, modTime time.Time, size int64, storable bool, hashes map[HashType]string, fs Info) ObjectInfo { + info := &staticObjectInfo{ + remote: remote, + modTime: modTime, + size: size, + storable: storable, + hashes: hashes, + fs: fs, + } + if fs != nil && hashes == nil { + set := fs.Hashes().Array() + info.hashes = make(map[HashType]string) + for _, ht := range set { + info.hashes[ht] = "" + } + } + return info +} + +type staticObjectInfo struct { + remote string + modTime time.Time + size int64 + storable bool + hashes map[HashType]string + fs Info +} + +func (i *staticObjectInfo) Fs() Info { return i.fs } +func (i *staticObjectInfo) Remote() string { return i.remote } +func (i *staticObjectInfo) ModTime() time.Time { return i.modTime } +func (i *staticObjectInfo) Size() int64 { return i.size } +func (i *staticObjectInfo) Storable() bool { return i.storable } +func (i *staticObjectInfo) Hash(h HashType) (string, error) { + if len(i.hashes) == 0 { + return "", ErrHashUnsupported + } + if hash, ok := i.hashes[h]; ok { + return hash, nil + } + return "", ErrHashUnsupported +} diff --git a/fs/limited.go b/fs/limited.go index fe9768ecb..f5c48be8d 100644 --- a/fs/limited.go +++ b/fs/limited.go @@ -71,12 +71,13 @@ func (f *Limited) NewFsObject(remote string) Object { // May create the object even if it returns an error - if so // will return the object and the error, otherwise will return // nil and the error -func (f *Limited) Put(in io.Reader, remote string, modTime time.Time, size int64) (Object, error) { +func (f *Limited) Put(in io.Reader, src ObjectInfo) (Object, error) { + remote := src.Remote() obj := f.NewFsObject(remote) if obj == nil { return nil, fmt.Errorf("Can't create %q in limited fs", remote) } - return obj, obj.Update(in, modTime, size) + return obj, obj.Update(in, src) } // Mkdir make the directory (container, bucket) diff --git a/fs/operations.go b/fs/operations.go index b957e2d7a..7aa602c54 100644 --- a/fs/operations.go +++ b/fs/operations.go @@ -221,10 +221,10 @@ tryAgain: if doUpdate { actionTaken = "Copied (updated existing)" - err = dst.Update(in, src.ModTime(), src.Size()) + err = dst.Update(in, src) } else { actionTaken = "Copied (new)" - dst, err = f.Put(in, src.Remote(), src.ModTime(), src.Size()) + dst, err = f.Put(in, src) } inErr = in.Close() } diff --git a/fs/operations_test.go b/fs/operations_test.go index d0f30412e..a2dfabff6 100644 --- a/fs/operations_test.go +++ b/fs/operations_test.go @@ -188,7 +188,8 @@ func (r *Run) WriteObjectTo(f fs.Fs, remote, content string, modTime time.Time) } for tries := 1; ; tries++ { in := bytes.NewBufferString(content) - _, err := f.Put(in, remote, modTime, int64(len(content))) + objinfo := fs.NewStaticObjectInfo(remote, modTime, int64(len(content)), true, nil, nil) + _, err := f.Put(in, objinfo) if err == nil { break } diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go index 006cae764..dcbd7a527 100644 --- a/fstest/fstests/fstests.go +++ b/fstest/fstests/fstests.go @@ -164,7 +164,8 @@ func testPut(t *testing.T, file *fstest.Item) { in := io.TeeReader(buf, hash) file.Size = int64(buf.Len()) - obj, err := remote.Put(in, file.Path, file.ModTime, file.Size) + obji := fs.NewStaticObjectInfo(file.Path, file.ModTime, file.Size, true, nil, nil) + obj, err := remote.Put(in, obji) if err != nil { t.Fatal("Put error", err) } @@ -551,7 +552,8 @@ func TestObjectUpdate(t *testing.T) { file1.Size = int64(buf.Len()) obj := findObject(t, file1.Path) - err := obj.Update(in, file1.ModTime, file1.Size) + obji := fs.NewStaticObjectInfo("", file1.ModTime, file1.Size, true, nil, obj.Fs()) + err := obj.Update(in, obji) if err != nil { t.Fatal("Update error", err) } diff --git a/googlecloudstorage/googlecloudstorage.go b/googlecloudstorage/googlecloudstorage.go index e81f8b96d..aa844e38b 100644 --- a/googlecloudstorage/googlecloudstorage.go +++ b/googlecloudstorage/googlecloudstorage.go @@ -55,7 +55,7 @@ var ( // Register with Fs func init() { - fs.Register(&fs.Info{ + fs.Register(&fs.RegInfo{ Name: "google cloud storage", NewFs: NewFs, Config: func(name string) { @@ -379,13 +379,13 @@ func (f *Fs) ListDir() fs.DirChan { // Copy the reader in to the new object which is returned // // The new object may have been created if an error is returned -func (f *Fs) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) { +func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) { // Temporary Object under construction o := &Object{ fs: f, - remote: remote, + remote: src.Remote(), } - return o, o.Update(in, modTime, size) + return o, o.Update(in, src) } // Mkdir creates the bucket if it doesn't exist @@ -466,7 +466,7 @@ func (f *Fs) Hashes() fs.HashSet { // ------------------------------------------------------------ // Fs returns the parent Fs -func (o *Object) Fs() fs.Fs { +func (o *Object) Fs() fs.Info { return o.fs } @@ -617,7 +617,10 @@ func (o *Object) Open() (in io.ReadCloser, err error) { // Update the object with the contents of the io.Reader, modTime and size // // The new object may have been created if an error is returned -func (o *Object) Update(in io.Reader, modTime time.Time, size int64) error { +func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error { + size := src.Size() + modTime := src.ModTime() + object := storage.Object{ Bucket: o.fs.bucket, Name: o.fs.root + o.remote, diff --git a/hubic/hubic.go b/hubic/hubic.go index aff747587..44752c575 100644 --- a/hubic/hubic.go +++ b/hubic/hubic.go @@ -44,7 +44,7 @@ var ( // Register with Fs func init() { - fs.Register(&fs.Info{ + fs.Register(&fs.RegInfo{ Name: "hubic", NewFs: NewFs, Config: func(name string) { diff --git a/local/local.go b/local/local.go index 6753a0e0b..a2f70845f 100644 --- a/local/local.go +++ b/local/local.go @@ -19,7 +19,7 @@ import ( // Register with Fs func init() { - fsi := &fs.Info{ + fsi := &fs.RegInfo{ Name: "local", NewFs: NewFs, Options: []fs.Option{fs.Option{ @@ -230,10 +230,11 @@ func (f *Fs) ListDir() fs.DirChan { } // Put the FsObject to the local filesystem -func (f *Fs) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) { +func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) { + remote := src.Remote() // Temporary FsObject under construction - info filled in by Update() o := f.newFsObject(remote) - err := o.Update(in, modTime, size) + err := o.Update(in, src) if err != nil { return nil, err } @@ -422,7 +423,7 @@ func (f *Fs) Hashes() fs.HashSet { // ------------------------------------------------------------ // Fs returns the parent Fs -func (o *Object) Fs() fs.Fs { +func (o *Object) Fs() fs.Info { return o.fs } @@ -568,7 +569,7 @@ func (o *Object) mkdirAll() error { } // Update the object from in with modTime and size -func (o *Object) Update(in io.Reader, modTime time.Time, size int64) error { +func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error { err := o.mkdirAll() if err != nil { return err @@ -579,7 +580,7 @@ func (o *Object) Update(in io.Reader, modTime time.Time, size int64) error { return err } - // Calculate the md5sum of the object we are reading as we go along + // Calculate the hash of the object we are reading as we go along hash := fs.NewMultiHasher() in = io.TeeReader(in, hash) @@ -596,7 +597,7 @@ func (o *Object) Update(in io.Reader, modTime time.Time, size int64) error { o.hashes = hash.Sums() // Set the mtime - o.SetModTime(modTime) + o.SetModTime(src.ModTime()) // ReRead info now that we have finished return o.lstat() diff --git a/onedrive/onedrive.go b/onedrive/onedrive.go index 89a495293..f7684ca74 100644 --- a/onedrive/onedrive.go +++ b/onedrive/onedrive.go @@ -55,7 +55,7 @@ var ( // Register with Fs func init() { - fs.Register(&fs.Info{ + fs.Register(&fs.RegInfo{ Name: "onedrive", NewFs: NewFs, Config: func(name string) { @@ -487,12 +487,16 @@ func (f *Fs) createObject(remote string, modTime time.Time, size int64) (o *Obje // Copy the reader in to the new object which is returned // // The new object may have been created if an error is returned -func (f *Fs) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) { +func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) { + remote := src.Remote() + size := src.Size() + modTime := src.ModTime() + o, _, _, err := f.createObject(remote, modTime, size) if err != nil { return nil, err } - return o, o.Update(in, modTime, size) + return o, o.Update(in, src) } // Mkdir creates the container if it doesn't exist @@ -679,7 +683,7 @@ func (f *Fs) Hashes() fs.HashSet { // ------------------------------------------------------------ // Fs returns the parent Fs -func (o *Object) Fs() fs.Fs { +func (o *Object) Fs() fs.Info { return o.fs } @@ -935,7 +939,10 @@ func (o *Object) uploadMultipart(in io.Reader, size int64) (err error) { // Update the object with the contents of the io.Reader, modTime and size // // The new object may have been created if an error is returned -func (o *Object) Update(in io.Reader, modTime time.Time, size int64) (err error) { +func (o *Object) Update(in io.Reader, src fs.ObjectInfo) (err error) { + size := src.Size() + modTime := src.ModTime() + var info *api.Item if size <= int64(uploadCutoff) { // This is for less than 100 MB of content diff --git a/s3/s3.go b/s3/s3.go index f728830f3..4334bc0c5 100644 --- a/s3/s3.go +++ b/s3/s3.go @@ -40,7 +40,7 @@ import ( // Register with Fs func init() { - fs.Register(&fs.Info{ + fs.Register(&fs.RegInfo{ Name: "s3", NewFs: NewFs, // AWS endpoints: http://docs.amazonwebservices.com/general/latest/gr/rande.html#s3_region @@ -495,13 +495,13 @@ func (f *Fs) ListDir() fs.DirChan { } // Put the FsObject into the bucket -func (f *Fs) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) { +func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) { // Temporary Object under construction fs := &Object{ fs: f, - remote: remote, + remote: src.Remote(), } - return fs, fs.Update(in, modTime, size) + return fs, fs.Update(in, src) } // Mkdir creates the bucket if it doesn't exist @@ -582,7 +582,7 @@ func (f *Fs) Hashes() fs.HashSet { // ------------------------------------------------------------ // Fs returns the parent Fs -func (o *Object) Fs() fs.Fs { +func (o *Object) Fs() fs.Info { return o.fs } @@ -732,7 +732,9 @@ func (o *Object) Open() (in io.ReadCloser, err error) { } // Update the Object from in with modTime and size -func (o *Object) Update(in io.Reader, modTime time.Time, size int64) error { +func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error { + modTime := src.ModTime() + uploader := s3manager.NewUploader(o.fs.ses, func(u *s3manager.Uploader) { u.Concurrency = 2 u.LeavePartsOnError = false diff --git a/swift/swift.go b/swift/swift.go index 89d803a83..8dddf97e8 100644 --- a/swift/swift.go +++ b/swift/swift.go @@ -29,7 +29,7 @@ var ( // Register with Fs func init() { - fs.Register(&fs.Info{ + fs.Register(&fs.RegInfo{ Name: "swift", NewFs: NewFs, Options: []fs.Option{{ @@ -380,13 +380,13 @@ func (f *Fs) ListDir() fs.DirChan { // Copy the reader in to the new object which is returned // // The new object may have been created if an error is returned -func (f *Fs) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) { +func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) { // Temporary Object under construction fs := &Object{ fs: f, - remote: remote, + remote: src.Remote(), } - return fs, fs.Update(in, modTime, size) + return fs, fs.Update(in, src) } // Mkdir creates the container if it doesn't exist @@ -457,7 +457,7 @@ func (f *Fs) Hashes() fs.HashSet { // ------------------------------------------------------------ // Fs returns the parent Fs -func (o *Object) Fs() fs.Fs { +func (o *Object) Fs() fs.Info { return o.fs } @@ -655,7 +655,10 @@ func (o *Object) updateChunks(in io.Reader, headers swift.Headers, size int64) ( // Update the object with the contents of the io.Reader, modTime and size // // The new object may have been created if an error is returned -func (o *Object) Update(in io.Reader, modTime time.Time, size int64) error { +func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error { + size := src.Size() + modTime := src.ModTime() + // Note whether this has a manifest before starting isManifest, err := o.isManifestFile() if err != nil { diff --git a/yandex/yandex.go b/yandex/yandex.go index 98f0e13ec..82277fc57 100644 --- a/yandex/yandex.go +++ b/yandex/yandex.go @@ -41,7 +41,7 @@ var ( // Register with Fs func init() { - fs.Register(&fs.Info{ + fs.Register(&fs.RegInfo{ Name: "yandex", NewFs: NewFs, Config: func(name string) { @@ -326,7 +326,11 @@ func (f *Fs) ListDir() fs.DirChan { // Copy the reader in to the new object which is returned // // The new object may have been created if an error is returned -func (f *Fs) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) { +func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) { + remote := src.Remote() + size := src.Size() + modTime := src.ModTime() + o := &Object{ fs: f, remote: remote, @@ -334,7 +338,7 @@ func (f *Fs) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs modTime: modTime, } //TODO maybe read metadata after upload to check if file uploaded successfully - return o, o.Update(in, modTime, size) + return o, o.Update(in, src) } // Mkdir creates the container if it doesn't exist @@ -390,7 +394,7 @@ func (f *Fs) Hashes() fs.HashSet { // ------------------------------------------------------------ // Fs returns the parent Fs -func (o *Object) Fs() fs.Fs { +func (o *Object) Fs() fs.Info { return o.fs } @@ -472,7 +476,10 @@ func (o *Object) remotePath() string { // Copy the reader into the object updating modTime and size // // The new object may have been created if an error is returned -func (o *Object) Update(in io.Reader, modTime time.Time, size int64) error { +func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error { + size := src.Size() + modTime := src.ModTime() + remote := o.remotePath() //create full path to file before upload. err1 := mkDirFullPath(o.fs.yd, remote) @@ -483,7 +490,7 @@ func (o *Object) Update(in io.Reader, modTime time.Time, size int64) error { overwrite := true //overwrite existing file err := o.fs.yd.Upload(in, remote, overwrite) if err == nil { - //if file uploaded sucessfuly then return metadata + //if file uploaded sucessfully then return metadata o.bytes = uint64(size) o.modTime = modTime o.md5sum = "" // according to unit tests after put the md5 is empty.