From 945f49ab5e57a5a115b960db3dcb785ab5e6dd32 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 21 Sep 2016 22:13:24 +0100 Subject: [PATCH] Make ContentType be preserved for cloud -> cloud copies - fixes #733 --- amazonclouddrive/amazonclouddrive.go | 11 ++- amazonclouddrive/amazonclouddrive_test.go | 1 + b2/b2.go | 28 ++++--- b2/b2_test.go | 1 + crypt/crypt2_test.go | 1 + crypt/crypt_test.go | 1 + docs/content/overview.md | 43 +++++++---- drive/drive.go | 19 ++++- drive/drive_test.go | 1 + dropbox/dropbox.go | 25 +++++-- dropbox/dropbox_test.go | 1 + fs/fs.go | 7 ++ fs/operations.go | 20 ++++- fstest/fstests/fstests.go | 16 ++++ googlecloudstorage/googlecloudstorage.go | 28 ++++--- googlecloudstorage/googlecloudstorage_test.go | 1 + hubic/hubic_test.go | 1 + local/local_test.go | 1 + onedrive/onedrive.go | 16 +++- onedrive/onedrive_test.go | 1 + s3/s3.go | 31 +++++--- s3/s3_test.go | 1 + swift/swift.go | 23 ++++-- swift/swift_test.go | 1 + yandex/api/performupload.go | 3 +- yandex/api/upload.go | 4 +- yandex/yandex.go | 74 ++++++++++++------- yandex/yandex_test.go | 1 + 28 files changed, 265 insertions(+), 96 deletions(-) diff --git a/amazonclouddrive/amazonclouddrive.go b/amazonclouddrive/amazonclouddrive.go index 2fc5b1276..9c4979b42 100644 --- a/amazonclouddrive/amazonclouddrive.go +++ b/amazonclouddrive/amazonclouddrive.go @@ -846,6 +846,14 @@ func (o *Object) Remove() error { return err } +// MimeType of an Object if known, "" otherwise +func (o *Object) MimeType() string { + if o.info.ContentProperties.ContentType != nil { + return *o.info.ContentProperties.ContentType + } + return "" +} + // Check the interfaces are satisfied var ( _ fs.Fs = (*Fs)(nil) @@ -853,5 +861,6 @@ var ( // _ fs.Copier = (*Fs)(nil) // _ fs.Mover = (*Fs)(nil) // _ fs.DirMover = (*Fs)(nil) - _ fs.Object = (*Object)(nil) + _ fs.Object = (*Object)(nil) + _ fs.MimeTyper = &Object{} ) diff --git a/amazonclouddrive/amazonclouddrive_test.go b/amazonclouddrive/amazonclouddrive_test.go index 39d4d9c41..3748d3f53 100644 --- a/amazonclouddrive/amazonclouddrive_test.go +++ b/amazonclouddrive/amazonclouddrive_test.go @@ -46,6 +46,7 @@ func TestObjectFs(t *testing.T) { fstests.TestObjectFs(t) } func TestObjectRemote(t *testing.T) { fstests.TestObjectRemote(t) } func TestObjectHashes(t *testing.T) { fstests.TestObjectHashes(t) } func TestObjectModTime(t *testing.T) { fstests.TestObjectModTime(t) } +func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) } func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } diff --git a/b2/b2.go b/b2/b2.go index fcdf8abf9..e0781ecf9 100644 --- a/b2/b2.go +++ b/b2/b2.go @@ -98,12 +98,13 @@ type Fs struct { // Object describes a b2 object type Object struct { - fs *Fs // what this object is part of - remote string // The remote path - id string // b2 id of the file - modTime time.Time // The modified time of the object if known - sha1 string // SHA-1 hash if known - size int64 // Size of the object + fs *Fs // what this object is part of + remote string // The remote path + id string // b2 id of the file + modTime time.Time // The modified time of the object if known + sha1 string // SHA-1 hash if known + size int64 // Size of the object + mimeType string // Content-Type of the object } // ------------------------------------------------------------ @@ -896,9 +897,10 @@ func (o *Object) Size() int64 { // o.modTime // o.size // o.sha1 -func (o *Object) decodeMetaDataRaw(ID, SHA1 string, Size int64, UploadTimestamp api.Timestamp, Info map[string]string) (err error) { +func (o *Object) decodeMetaDataRaw(ID, SHA1 string, Size int64, UploadTimestamp api.Timestamp, Info map[string]string, mimeType string) (err error) { o.id = ID o.sha1 = SHA1 + o.mimeType = mimeType // Read SHA1 from metadata if it exists and isn't set if o.sha1 == "" || o.sha1 == "none" { o.sha1 = Info[sha1Key] @@ -917,7 +919,7 @@ func (o *Object) decodeMetaDataRaw(ID, SHA1 string, Size int64, UploadTimestamp // o.size // o.sha1 func (o *Object) decodeMetaData(info *api.File) (err error) { - return o.decodeMetaDataRaw(info.ID, info.SHA1, info.Size, info.UploadTimestamp, info.Info) + return o.decodeMetaDataRaw(info.ID, info.SHA1, info.Size, info.UploadTimestamp, info.Info, info.ContentType) } // decodeMetaDataFileInfo sets the metadata in the object from an api.FileInfo @@ -928,7 +930,7 @@ func (o *Object) decodeMetaData(info *api.File) (err error) { // o.size // o.sha1 func (o *Object) decodeMetaDataFileInfo(info *api.FileInfo) (err error) { - return o.decodeMetaDataRaw(info.ID, info.SHA1, info.Size, info.UploadTimestamp, info.Info) + return o.decodeMetaDataRaw(info.ID, info.SHA1, info.Size, info.UploadTimestamp, info.Info, info.ContentType) } // readMetaData gets the metadata if it hasn't already been fetched @@ -1285,7 +1287,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo) (err error) { ExtraHeaders: map[string]string{ "Authorization": upload.AuthorizationToken, "X-Bz-File-Name": urlEncode(o.fs.root + o.remote), - "Content-Type": fs.MimeType(o), + "Content-Type": fs.MimeType(src), sha1Header: calculatedSha1, timeHeader: timeString(modTime), }, @@ -1337,10 +1339,16 @@ func (o *Object) Remove() error { return nil } +// MimeType of an Object if known, "" otherwise +func (o *Object) MimeType() string { + return o.mimeType +} + // Check the interfaces are satisfied var ( _ fs.Fs = &Fs{} _ fs.Purger = &Fs{} _ fs.CleanUpper = &Fs{} _ fs.Object = &Object{} + _ fs.MimeTyper = &Object{} ) diff --git a/b2/b2_test.go b/b2/b2_test.go index 64c45533a..fb3f1c808 100644 --- a/b2/b2_test.go +++ b/b2/b2_test.go @@ -46,6 +46,7 @@ func TestObjectFs(t *testing.T) { fstests.TestObjectFs(t) } func TestObjectRemote(t *testing.T) { fstests.TestObjectRemote(t) } func TestObjectHashes(t *testing.T) { fstests.TestObjectHashes(t) } func TestObjectModTime(t *testing.T) { fstests.TestObjectModTime(t) } +func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) } func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } diff --git a/crypt/crypt2_test.go b/crypt/crypt2_test.go index 9fd7db801..25ae95323 100644 --- a/crypt/crypt2_test.go +++ b/crypt/crypt2_test.go @@ -47,6 +47,7 @@ func TestObjectFs2(t *testing.T) { fstests.TestObjectFs(t) } func TestObjectRemote2(t *testing.T) { fstests.TestObjectRemote(t) } func TestObjectHashes2(t *testing.T) { fstests.TestObjectHashes(t) } func TestObjectModTime2(t *testing.T) { fstests.TestObjectModTime(t) } +func TestObjectMimeType2(t *testing.T) { fstests.TestObjectMimeType(t) } func TestObjectSetModTime2(t *testing.T) { fstests.TestObjectSetModTime(t) } func TestObjectSize2(t *testing.T) { fstests.TestObjectSize(t) } func TestObjectOpen2(t *testing.T) { fstests.TestObjectOpen(t) } diff --git a/crypt/crypt_test.go b/crypt/crypt_test.go index f994fa918..e20ad1fa8 100644 --- a/crypt/crypt_test.go +++ b/crypt/crypt_test.go @@ -47,6 +47,7 @@ func TestObjectFs(t *testing.T) { fstests.TestObjectFs(t) } func TestObjectRemote(t *testing.T) { fstests.TestObjectRemote(t) } func TestObjectHashes(t *testing.T) { fstests.TestObjectHashes(t) } func TestObjectModTime(t *testing.T) { fstests.TestObjectModTime(t) } +func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) } func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } diff --git a/docs/content/overview.md b/docs/content/overview.md index 26bb0d319..5fbbee92e 100644 --- a/docs/content/overview.md +++ b/docs/content/overview.md @@ -15,19 +15,19 @@ show through. Here is an overview of the major features of each cloud storage system. -| Name | Hash | ModTime | Case Insensitive | Duplicate Files | -| ---------------------- |:-------:|:-------:|:----------------:|:---------------:| -| Google Drive | MD5 | Yes | No | Yes | -| Amazon S3 | MD5 | Yes | No | No | -| Openstack Swift | MD5 | Yes | No | No | -| Dropbox | - | No | Yes | No | -| Google Cloud Storage | MD5 | Yes | No | No | -| Amazon Drive | MD5 | No | Yes | No | -| Microsoft One Drive | SHA1 | Yes | Yes | No | -| Hubic | MD5 | Yes | No | No | -| Backblaze B2 | SHA1 | Yes | No | No | -| Yandex Disk | MD5 | Yes | No | No | -| The local filesystem | All | Yes | Depends | No | +| Name | Hash | ModTime | Case Insensitive | Duplicate Files | MIME Type | +| ---------------------- |:-------:|:-------:|:----------------:|:---------------:|:---------:| +| Google Drive | MD5 | Yes | No | Yes | R/W | +| Amazon S3 | MD5 | Yes | No | No | R/W | +| Openstack Swift | MD5 | Yes | No | No | R/W | +| Dropbox | - | No | Yes | No | R | +| Google Cloud Storage | MD5 | Yes | No | No | R/W | +| Amazon Drive | MD5 | No | Yes | No | R | +| Microsoft One Drive | SHA1 | Yes | Yes | No | R | +| Hubic | MD5 | Yes | No | No | R/W | +| Backblaze B2 | SHA1 | Yes | No | No | R/W | +| Yandex Disk | MD5 | Yes | No | No | R/W | +| The local filesystem | All | Yes | Depends | No | - | ### Hash ### @@ -78,6 +78,23 @@ objects with the same name. This confuses rclone greatly when syncing - use the `rclone dedupe` command to rename or remove duplicates. +### MIME Type ### + +MIME types (also known as media types) classify types of documents +using a simple text classification, eg `text/html` or +`application/pdf`. + +Some cloud storage systems support reading (`R`) the MIME type of +objects and some support writing (`W`) the MIME type of objects. + +The MIME type can be important if you are serving files directly to +HTTP from the storage system. + +If you are copying from a remote which supports reading (`R`) to a +remote which supports writing (`W`) then rclone will preserve the MIME +types. Otherwise they will be guessed from the extension, or the +remote itself may assign the MIME type. + ## Optional Features ## All the remotes support a basic set of features, but there are some diff --git a/drive/drive.go b/drive/drive.go index 85de3ab0a..632b0e070 100644 --- a/drive/drive.go +++ b/drive/drive.go @@ -137,6 +137,7 @@ type Object struct { bytes int64 // size of the object modifiedDate string // RFC3339 time it was last modified isDocument bool // if set this is a Google doc + mimeType string } // ------------------------------------------------------------ @@ -533,7 +534,7 @@ func (f *Fs) createFileInfo(remote string, modTime time.Time, size int64) (*Obje Title: leaf, Description: leaf, Parents: []*drive.ParentReference{{Id: directoryID}}, - MimeType: fs.MimeType(o), + MimeType: fs.MimeTypeFromName(remote), ModifiedDate: modTime.Format(timeFormatOut), } return o, createInfo, nil @@ -845,6 +846,7 @@ func (o *Object) setMetaData(info *drive.File) { o.md5sum = strings.ToLower(info.Md5Checksum) o.bytes = info.FileSize o.modifiedDate = info.ModifiedDate + o.mimeType = info.MimeType } // readMetaData gets the info if it hasn't already been fetched @@ -1009,7 +1011,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error { } updateInfo := &drive.File{ Id: o.id, - MimeType: fs.MimeType(o), + MimeType: fs.MimeType(src), ModifiedDate: modTime.Format(timeFormatOut), } @@ -1027,7 +1029,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error { } } else { // Upload the file in chunks - info, err = o.fs.Upload(in, size, fs.MimeType(o), updateInfo, o.remote) + info, err = o.fs.Upload(in, size, updateInfo.MimeType, updateInfo, o.remote) if err != nil { return err } @@ -1053,6 +1055,16 @@ func (o *Object) Remove() error { return err } +// MimeType of an Object if known, "" otherwise +func (o *Object) MimeType() string { + err := o.readMetaData() + if err != nil { + fs.Log(o, "Failed to read metadata: %v", err) + return "" + } + return o.mimeType +} + // Check the interfaces are satisfied var ( _ fs.Fs = (*Fs)(nil) @@ -1062,4 +1074,5 @@ var ( _ fs.DirMover = (*Fs)(nil) _ fs.PutUncheckeder = (*Fs)(nil) _ fs.Object = (*Object)(nil) + _ fs.MimeTyper = &Object{} ) diff --git a/drive/drive_test.go b/drive/drive_test.go index 657a78a1e..e7f790f86 100644 --- a/drive/drive_test.go +++ b/drive/drive_test.go @@ -46,6 +46,7 @@ func TestObjectFs(t *testing.T) { fstests.TestObjectFs(t) } func TestObjectRemote(t *testing.T) { fstests.TestObjectRemote(t) } func TestObjectHashes(t *testing.T) { fstests.TestObjectHashes(t) } func TestObjectModTime(t *testing.T) { fstests.TestObjectModTime(t) } +func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) } func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } diff --git a/dropbox/dropbox.go b/dropbox/dropbox.go index 7c6231823..4390b77bd 100644 --- a/dropbox/dropbox.go +++ b/dropbox/dropbox.go @@ -110,6 +110,7 @@ type Object struct { bytes int64 // size of the object modTime time.Time // time it was last modified hasMetadata bool // metadata is valid + mimeType string // content type according to the server } // ------------------------------------------------------------ @@ -622,6 +623,7 @@ func (o *Object) Size() int64 { func (o *Object) setMetadataFromEntry(info *dropbox.Entry) { o.bytes = info.Bytes o.modTime = time.Time(info.ClientMtime) + o.mimeType = info.MimeType o.hasMetadata = true } @@ -745,12 +747,23 @@ func (o *Object) Remove() error { return err } +// MimeType of an Object if known, "" otherwise +func (o *Object) MimeType() string { + err := o.readMetaData() + if err != nil { + fs.Log(o, "Failed to read metadata: %v", err) + return "" + } + return o.mimeType +} + // Check the interfaces are satisfied var ( - _ fs.Fs = (*Fs)(nil) - _ fs.Copier = (*Fs)(nil) - _ fs.Purger = (*Fs)(nil) - _ fs.Mover = (*Fs)(nil) - _ fs.DirMover = (*Fs)(nil) - _ fs.Object = (*Object)(nil) + _ fs.Fs = (*Fs)(nil) + _ fs.Copier = (*Fs)(nil) + _ fs.Purger = (*Fs)(nil) + _ fs.Mover = (*Fs)(nil) + _ fs.DirMover = (*Fs)(nil) + _ fs.Object = (*Object)(nil) + _ fs.MimeTyper = (*Object)(nil) ) diff --git a/dropbox/dropbox_test.go b/dropbox/dropbox_test.go index 59c68e659..386949d46 100644 --- a/dropbox/dropbox_test.go +++ b/dropbox/dropbox_test.go @@ -46,6 +46,7 @@ func TestObjectFs(t *testing.T) { fstests.TestObjectFs(t) } func TestObjectRemote(t *testing.T) { fstests.TestObjectRemote(t) } func TestObjectHashes(t *testing.T) { fstests.TestObjectHashes(t) } func TestObjectModTime(t *testing.T) { fstests.TestObjectModTime(t) } +func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) } func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } diff --git a/fs/fs.go b/fs/fs.go index 307365e25..0f981be0a 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -210,6 +210,13 @@ type BasicInfo interface { Size() int64 } +// MimeTyper is an optional interface for Object +type MimeTyper interface { + // MimeType returns the content type of the Object if + // known, or "" if not + MimeType() string +} + // Purger is an optional interfaces for Fs type Purger interface { // Purge all files in the root and the root directory diff --git a/fs/operations.go b/fs/operations.go index c6aad3c71..d81a8a319 100644 --- a/fs/operations.go +++ b/fs/operations.go @@ -169,15 +169,29 @@ func Equal(src, dst Object) bool { return true } -// MimeType returns a guess at the mime type from the extension -func MimeType(o ObjectInfo) string { - mimeType := mime.TypeByExtension(path.Ext(o.Remote())) +// MimeTypeFromName returns a guess at the mime type from the name +func MimeTypeFromName(remote string) (mimeType string) { + mimeType = mime.TypeByExtension(path.Ext(remote)) if !strings.ContainsRune(mimeType, '/') { mimeType = "application/octet-stream" } return mimeType } +// MimeType returns the MimeType from the object, either by calling +// the MimeTyper interface or using MimeTypeFromName +func MimeType(o ObjectInfo) (mimeType string) { + // Read the MimeType from the optional interface if available + if do, ok := o.(MimeTyper); ok { + mimeType = do.MimeType() + Debug(o, "Read MimeType as %q", mimeType) + if mimeType != "" { + return mimeType + } + } + return MimeTypeFromName(o.Remote()) +} + // Used to remove a failed copy // // Returns whether the file was succesfully removed or not diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go index 24aaf1f89..0d6df66e6 100644 --- a/fstest/fstests/fstests.go +++ b/fstest/fstests/fstests.go @@ -501,6 +501,22 @@ func TestObjectModTime(t *testing.T) { file1.CheckModTime(t, obj, obj.ModTime(), remote.Precision()) } +// TestObjectMimeType tests the MimeType of the object is correct +func TestObjectMimeType(t *testing.T) { + skipIfNotOk(t) + obj := findObject(t, file1.Path) + do, ok := obj.(fs.MimeTyper) + if !ok { + t.Skip("MimeType method not supported") + } + mimeType := do.MimeType() + if strings.ContainsRune(mimeType, ';') { + assert.Equal(t, "text/plain; charset=utf-8", mimeType) + } else { + assert.Equal(t, "text/plain", mimeType) + } +} + // TestObjectSetModTime tests that SetModTime works func TestObjectSetModTime(t *testing.T) { skipIfNotOk(t) diff --git a/googlecloudstorage/googlecloudstorage.go b/googlecloudstorage/googlecloudstorage.go index f3a81e450..fe8675b28 100644 --- a/googlecloudstorage/googlecloudstorage.go +++ b/googlecloudstorage/googlecloudstorage.go @@ -143,12 +143,13 @@ type Fs struct { // // Will definitely have info but maybe not meta type Object struct { - fs *Fs // what this object is part of - remote string // The remote path - url string // download path - md5sum string // The MD5Sum of the object - bytes int64 // Bytes in the object - modTime time.Time // Modified time of the object + fs *Fs // what this object is part of + remote string // The remote path + url string // download path + md5sum string // The MD5Sum of the object + bytes int64 // Bytes in the object + modTime time.Time // Modified time of the object + mimeType string } // ------------------------------------------------------------ @@ -558,6 +559,7 @@ func (o *Object) Size() int64 { func (o *Object) setMetaData(info *storage.Object) { o.url = info.MediaLink o.bytes = int64(info.Size) + o.mimeType = info.ContentType // Read md5sum md5sumData, err := base64.StdEncoding.DecodeString(info.Md5Hash) @@ -675,7 +677,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error { object := storage.Object{ Bucket: o.fs.bucket, Name: o.fs.root + o.remote, - ContentType: fs.MimeType(o), + ContentType: fs.MimeType(src), Size: uint64(size), Updated: modTime.Format(timeFormatOut), // Doesn't get set Metadata: metadataFromModTime(modTime), @@ -694,9 +696,15 @@ func (o *Object) Remove() error { return o.fs.svc.Objects.Delete(o.fs.bucket, o.fs.root+o.remote).Do() } +// MimeType of an Object if known, "" otherwise +func (o *Object) MimeType() string { + return o.mimeType +} + // Check the interfaces are satisfied var ( - _ fs.Fs = &Fs{} - _ fs.Copier = &Fs{} - _ fs.Object = &Object{} + _ fs.Fs = &Fs{} + _ fs.Copier = &Fs{} + _ fs.Object = &Object{} + _ fs.MimeTyper = &Object{} ) diff --git a/googlecloudstorage/googlecloudstorage_test.go b/googlecloudstorage/googlecloudstorage_test.go index 208435376..acbe1508a 100644 --- a/googlecloudstorage/googlecloudstorage_test.go +++ b/googlecloudstorage/googlecloudstorage_test.go @@ -46,6 +46,7 @@ func TestObjectFs(t *testing.T) { fstests.TestObjectFs(t) } func TestObjectRemote(t *testing.T) { fstests.TestObjectRemote(t) } func TestObjectHashes(t *testing.T) { fstests.TestObjectHashes(t) } func TestObjectModTime(t *testing.T) { fstests.TestObjectModTime(t) } +func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) } func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } diff --git a/hubic/hubic_test.go b/hubic/hubic_test.go index 46eda5725..7159b0b82 100644 --- a/hubic/hubic_test.go +++ b/hubic/hubic_test.go @@ -46,6 +46,7 @@ func TestObjectFs(t *testing.T) { fstests.TestObjectFs(t) } func TestObjectRemote(t *testing.T) { fstests.TestObjectRemote(t) } func TestObjectHashes(t *testing.T) { fstests.TestObjectHashes(t) } func TestObjectModTime(t *testing.T) { fstests.TestObjectModTime(t) } +func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) } func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } diff --git a/local/local_test.go b/local/local_test.go index 4619032cc..a4c9c4b00 100644 --- a/local/local_test.go +++ b/local/local_test.go @@ -46,6 +46,7 @@ func TestObjectFs(t *testing.T) { fstests.TestObjectFs(t) } func TestObjectRemote(t *testing.T) { fstests.TestObjectRemote(t) } func TestObjectHashes(t *testing.T) { fstests.TestObjectHashes(t) } func TestObjectModTime(t *testing.T) { fstests.TestObjectModTime(t) } +func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) } func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } diff --git a/onedrive/onedrive.go b/onedrive/onedrive.go index ec8bf308c..e870c6377 100644 --- a/onedrive/onedrive.go +++ b/onedrive/onedrive.go @@ -98,6 +98,7 @@ type Object struct { modTime time.Time // modification time of the object id string // ID of the object sha1 string // SHA-1 of the object content + mimeType string // Content-Type of object from server (may not be as uploaded) } // ------------------------------------------------------------ @@ -686,8 +687,11 @@ func (o *Object) setMetaData(info *api.Item) { // fact uppercase hex strings. // // In OneDrive for Business, SHA1 and CRC32 hash values are not returned for files. - if info.File != nil && info.File.Hashes.Sha1Hash != "" { - o.sha1 = strings.ToLower(info.File.Hashes.Sha1Hash) + if info.File != nil { + o.mimeType = info.File.MimeType + if info.File.Hashes.Sha1Hash != "" { + o.sha1 = strings.ToLower(info.File.Hashes.Sha1Hash) + } } if info.FileSystemInfo != nil { o.modTime = time.Time(info.FileSystemInfo.LastModifiedDateTime) @@ -935,6 +939,11 @@ func (o *Object) Remove() error { return o.fs.deleteObject(o.id) } +// MimeType of an Object if known, "" otherwise +func (o *Object) MimeType() string { + return o.mimeType +} + // Check the interfaces are satisfied var ( _ fs.Fs = (*Fs)(nil) @@ -942,5 +951,6 @@ var ( _ fs.Copier = (*Fs)(nil) // _ fs.Mover = (*Fs)(nil) // _ fs.DirMover = (*Fs)(nil) - _ fs.Object = (*Object)(nil) + _ fs.Object = (*Object)(nil) + _ fs.MimeTyper = &Object{} ) diff --git a/onedrive/onedrive_test.go b/onedrive/onedrive_test.go index e8e230056..0c3807c91 100644 --- a/onedrive/onedrive_test.go +++ b/onedrive/onedrive_test.go @@ -46,6 +46,7 @@ func TestObjectFs(t *testing.T) { fstests.TestObjectFs(t) } func TestObjectRemote(t *testing.T) { fstests.TestObjectRemote(t) } func TestObjectHashes(t *testing.T) { fstests.TestObjectHashes(t) } func TestObjectModTime(t *testing.T) { fstests.TestObjectModTime(t) } +func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) } func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } diff --git a/s3/s3.go b/s3/s3.go index 4f01172ee..c8d4ef597 100644 --- a/s3/s3.go +++ b/s3/s3.go @@ -230,14 +230,15 @@ type Fs struct { type Object struct { // Will definitely have everything but meta which may be nil // - // List will read everything but meta - to fill that in need to call - // readMetaData + // List will read everything but meta & mimeType - to fill + // that in you need to call readMetaData fs *Fs // what this object is part of remote string // The remote path etag string // md5sum of the object bytes int64 // size of the object lastModified time.Time // Last modified meta map[string]*string // The object metadata if known - may be nil + mimeType string // MimeType of object - may be "" } // ------------------------------------------------------------ @@ -777,6 +778,7 @@ func (o *Object) readMetaData() (err error) { } else { o.lastModified = *resp.LastModified } + o.mimeType = aws.StringValue(resp.ContentType) return nil } @@ -818,7 +820,7 @@ func (o *Object) SetModTime(modTime time.Time) error { } // Guess the content type - contentType := fs.MimeType(o) + mimeType := fs.MimeType(o) // Copy the object to itself to update the metadata key := o.fs.root + o.remote @@ -828,7 +830,7 @@ func (o *Object) SetModTime(modTime time.Time) error { Bucket: &o.fs.bucket, ACL: &o.fs.acl, Key: &key, - ContentType: &contentType, + ContentType: &mimeType, CopySource: aws.String(url.QueryEscape(sourceKey)), Metadata: o.meta, MetadataDirective: &directive, @@ -880,7 +882,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error { } // Guess the content type - contentType := fs.MimeType(o) + mimeType := fs.MimeType(src) key := o.fs.root + o.remote req := s3manager.UploadInput{ @@ -888,7 +890,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error { ACL: &o.fs.acl, Key: &key, Body: in, - ContentType: &contentType, + ContentType: &mimeType, Metadata: metadata, //ContentLength: &size, } @@ -920,9 +922,20 @@ func (o *Object) Remove() error { return err } +// MimeType of an Object if known, "" otherwise +func (o *Object) MimeType() string { + err := o.readMetaData() + if err != nil { + fs.Log(o, "Failed to read metadata: %v", err) + return "" + } + return o.mimeType +} + // Check the interfaces are satisfied var ( - _ fs.Fs = &Fs{} - _ fs.Copier = &Fs{} - _ fs.Object = &Object{} + _ fs.Fs = &Fs{} + _ fs.Copier = &Fs{} + _ fs.Object = &Object{} + _ fs.MimeTyper = &Object{} ) diff --git a/s3/s3_test.go b/s3/s3_test.go index 5bde96ae9..6c6ec7426 100644 --- a/s3/s3_test.go +++ b/s3/s3_test.go @@ -46,6 +46,7 @@ func TestObjectFs(t *testing.T) { fstests.TestObjectFs(t) } func TestObjectRemote(t *testing.T) { fstests.TestObjectRemote(t) } func TestObjectHashes(t *testing.T) { fstests.TestObjectHashes(t) } func TestObjectModTime(t *testing.T) { fstests.TestObjectModTime(t) } +func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) } func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } diff --git a/swift/swift.go b/swift/swift.go index ea86b5e5b..32dcc0599 100644 --- a/swift/swift.go +++ b/swift/swift.go @@ -689,7 +689,7 @@ func urlEncode(str string) string { // updateChunks updates the existing object using chunks to a separate // container. It returns a string which prefixes current segments. -func (o *Object) updateChunks(in io.Reader, headers swift.Headers, size int64) (string, error) { +func (o *Object) updateChunks(in io.Reader, headers swift.Headers, size int64, contentType string) (string, error) { // Create the segmentsContainer if it doesn't exist err := o.fs.c.ContainerCreate(o.fs.segmentsContainer, nil) if err != nil { @@ -718,7 +718,7 @@ func (o *Object) updateChunks(in io.Reader, headers swift.Headers, size int64) ( headers["Content-Length"] = "0" // set Content-Length as we know it emptyReader := bytes.NewReader(nil) manifestName := o.fs.root + o.remote - _, err = o.fs.c.ObjectPut(o.fs.container, manifestName, emptyReader, true, "", "", headers) + _, err = o.fs.c.ObjectPut(o.fs.container, manifestName, emptyReader, true, "", contentType, headers) return uniquePrefix + "/", err } @@ -738,16 +738,17 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error { // Set the mtime m := swift.Metadata{} m.SetModTime(modTime) + contentType := fs.MimeType(src) headers := m.ObjectHeaders() uniquePrefix := "" if size > int64(chunkSize) { - uniquePrefix, err = o.updateChunks(in, headers, size) + uniquePrefix, err = o.updateChunks(in, headers, size, contentType) if err != nil { return err } } else { headers["Content-Length"] = strconv.FormatInt(size, 10) // set Content-Length as we know it - _, err := o.fs.c.ObjectPut(o.fs.container, o.fs.root+o.remote, in, true, "", "", headers) + _, err := o.fs.c.ObjectPut(o.fs.container, o.fs.root+o.remote, in, true, "", contentType, headers) if err != nil { return err } @@ -787,10 +788,16 @@ func (o *Object) Remove() error { return nil } +// MimeType of an Object if known, "" otherwise +func (o *Object) MimeType() string { + return o.info.ContentType +} + // Check the interfaces are satisfied var ( - _ fs.Fs = &Fs{} - _ fs.Purger = &Fs{} - _ fs.Copier = &Fs{} - _ fs.Object = &Object{} + _ fs.Fs = &Fs{} + _ fs.Purger = &Fs{} + _ fs.Copier = &Fs{} + _ fs.Object = &Object{} + _ fs.MimeTyper = &Object{} ) diff --git a/swift/swift_test.go b/swift/swift_test.go index 941c6e7b3..1bc0c56e0 100644 --- a/swift/swift_test.go +++ b/swift/swift_test.go @@ -46,6 +46,7 @@ func TestObjectFs(t *testing.T) { fstests.TestObjectFs(t) } func TestObjectRemote(t *testing.T) { fstests.TestObjectRemote(t) } func TestObjectHashes(t *testing.T) { fstests.TestObjectHashes(t) } func TestObjectModTime(t *testing.T) { fstests.TestObjectModTime(t) } +func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) } func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } diff --git a/yandex/api/performupload.go b/yandex/api/performupload.go index cfedb93d7..39f55cdd7 100644 --- a/yandex/api/performupload.go +++ b/yandex/api/performupload.go @@ -11,11 +11,12 @@ import ( ) // PerformUpload does the actual upload via unscoped PUT request. -func (c *Client) PerformUpload(url string, data io.Reader) (err error) { +func (c *Client) PerformUpload(url string, data io.Reader, contentType string) (err error) { req, err := http.NewRequest("PUT", url, data) if err != nil { return err } + req.Header.Set("Content-Type", contentType) //c.setRequestScope(req) diff --git a/yandex/api/upload.go b/yandex/api/upload.go index b2eff2f84..946e6592d 100644 --- a/yandex/api/upload.go +++ b/yandex/api/upload.go @@ -17,13 +17,13 @@ type UploadResponse struct { } // Upload will put specified data to Yandex.Disk. -func (c *Client) Upload(data io.Reader, remotePath string, overwrite bool) error { +func (c *Client) Upload(data io.Reader, remotePath string, overwrite bool, contentType string) error { ur, err := c.UploadRequest(remotePath, overwrite) if err != nil { return err } - if err := c.PerformUpload(ur.HRef, data); err != nil { + if err := c.PerformUpload(ur.HRef, data, contentType); err != nil { return err } diff --git a/yandex/yandex.go b/yandex/yandex.go index 93c46aa1d..1e7c10509 100644 --- a/yandex/yandex.go +++ b/yandex/yandex.go @@ -73,11 +73,12 @@ type Fs struct { // Object describes a swift object type Object struct { - fs *Fs // what this object is part of - remote string // The remote path - md5sum string // The MD5Sum of the object - bytes uint64 // Bytes in the object - modTime time.Time // Modified time of the object + fs *Fs // what this object is part of + remote string // The remote path + md5sum string // The MD5Sum of the object + bytes uint64 // Bytes in the object + modTime time.Time // Modified time of the object + mimeType string // Content type according to the server } // ------------------------------------------------------------ @@ -326,31 +327,33 @@ func (f *Fs) newObjectWithInfo(remote string, info *yandex.ResourceInfoResponse) func (o *Object) setMetaData(info *yandex.ResourceInfoResponse) { o.bytes = info.Size o.md5sum = info.Md5 + o.mimeType = info.MimeType - if info.CustomProperties["rclone_modified"] == nil { - //read modTime from Modified property of object - t, err := time.Parse(time.RFC3339Nano, info.Modified) - if err != nil { - return - } - o.modTime = t + var modTimeString string + modTimeObj, ok := info.CustomProperties["rclone_modified"] + if ok { + // read modTime from rclone_modified custom_property of object + modTimeString, ok = modTimeObj.(string) + } + if !ok { + // read modTime from Modified property of object as a fallback + modTimeString = info.Modified + } + t, err := time.Parse(time.RFC3339Nano, modTimeString) + if err != nil { + fs.Log("Failed to parse modtime from %q: %v", modTimeString, err) } else { - // interface{} to string type assertion - if modtimestr, ok := info.CustomProperties["rclone_modified"].(string); ok { - //read modTime from rclone_modified custom_property of object - t, err := time.Parse(time.RFC3339Nano, modtimestr) - if err != nil { - return - } - o.modTime = t - } else { - return //if it is not a string - } + o.modTime = t } } // readMetaData gets the info if it hasn't already been fetched func (o *Object) readMetaData() (err error) { + // exit if already fetched + if !o.modTime.IsZero() { + return nil + } + //request meta info var opt2 yandex.ResourceInfoRequestOptions ResourceInfoResponse, err := o.fs.yd.NewResourceInfoRequest(o.remotePath(), opt2).Exec() @@ -498,8 +501,13 @@ func (o *Object) Remove() error { // Commits the datastore func (o *Object) SetModTime(modTime time.Time) error { remote := o.remotePath() - //set custom_property 'rclone_modified' of object to modTime - return o.fs.yd.SetCustomProperty(remote, "rclone_modified", modTime.Format(time.RFC3339Nano)) + // set custom_property 'rclone_modified' of object to modTime + err := o.fs.yd.SetCustomProperty(remote, "rclone_modified", modTime.Format(time.RFC3339Nano)) + if err != nil { + return err + } + o.modTime = modTime + return nil } // Storable returns whether this object is storable @@ -529,7 +537,8 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error { } //upload file overwrite := true //overwrite existing file - err := o.fs.yd.Upload(in, remote, overwrite) + mimeType := fs.MimeType(src) + err := o.fs.yd.Upload(in, remote, overwrite, mimeType) if err == nil { //if file uploaded sucessfully then return metadata o.bytes = uint64(size) @@ -587,10 +596,21 @@ func mkDirFullPath(client *yandex.Client, path string) error { return nil } +// MimeType of an Object if known, "" otherwise +func (o *Object) MimeType() string { + err := o.readMetaData() + if err != nil { + fs.Log(o, "Failed to read metadata: %v", err) + return "" + } + return o.mimeType +} + // Check the interfaces are satisfied var ( _ fs.Fs = (*Fs)(nil) _ fs.Purger = (*Fs)(nil) //_ fs.Copier = (*Fs)(nil) - _ fs.Object = (*Object)(nil) + _ fs.Object = (*Object)(nil) + _ fs.MimeTyper = &Object{} ) diff --git a/yandex/yandex_test.go b/yandex/yandex_test.go index 23ef7a2a0..235170d3a 100644 --- a/yandex/yandex_test.go +++ b/yandex/yandex_test.go @@ -46,6 +46,7 @@ func TestObjectFs(t *testing.T) { fstests.TestObjectFs(t) } func TestObjectRemote(t *testing.T) { fstests.TestObjectRemote(t) } func TestObjectHashes(t *testing.T) { fstests.TestObjectHashes(t) } func TestObjectModTime(t *testing.T) { fstests.TestObjectModTime(t) } +func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) } func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }