From 19ad39fa1c2cdd1ea44f9cd115f47524a45d3dce Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Thu, 4 Aug 2022 20:48:13 +0200 Subject: [PATCH] jottacloud: add support for reading and writing metadata Most useful is the addition of the file created timestamp, but also a timestamp for when the file was uploaded. Currently supporting a rather minimalistic set of metadata items, see PR #6359 for some thoughts about possible extensions. --- backend/jottacloud/api/types.go | 2 +- backend/jottacloud/jottacloud.go | 122 +++++++++++++++--- .../jottacloud/jottacloud_internal_test.go | 51 ++++++++ docs/content/jottacloud.md | 7 +- docs/content/overview.md | 2 +- 5 files changed, 158 insertions(+), 26 deletions(-) diff --git a/backend/jottacloud/api/types.go b/backend/jottacloud/api/types.go index 8daf0f7ad..6d192bd4e 100644 --- a/backend/jottacloud/api/types.go +++ b/backend/jottacloud/api/types.go @@ -395,7 +395,7 @@ type JottaFile struct { State string `xml:"currentRevision>state"` CreatedAt JottaTime `xml:"currentRevision>created"` ModifiedAt JottaTime `xml:"currentRevision>modified"` - Updated JottaTime `xml:"currentRevision>updated"` + UpdatedAt JottaTime `xml:"currentRevision>updated"` Size int64 `xml:"currentRevision>size"` MimeType string `xml:"currentRevision>mime"` MD5 string `xml:"currentRevision>md5"` diff --git a/backend/jottacloud/jottacloud.go b/backend/jottacloud/jottacloud.go index e3f43c2b9..7c48c116d 100644 --- a/backend/jottacloud/jottacloud.go +++ b/backend/jottacloud/jottacloud.go @@ -92,6 +92,33 @@ func init() { Description: "Jottacloud", NewFs: NewFs, Config: Config, + MetadataInfo: &fs.MetadataInfo{ + Help: `Jottacloud has limited support for metadata, currently an extended set of timestamps.`, + System: map[string]fs.MetadataHelp{ + "btime": { + Help: "Time of file birth (creation), read from rclone metadata", + Type: "RFC 3339", + Example: "2006-01-02T15:04:05.999999999Z07:00", + }, + "mtime": { + Help: "Time of last modification, read from rclone metadata", + Type: "RFC 3339", + Example: "2006-01-02T15:04:05.999999999Z07:00", + }, + "utime": { + Help: "Time of last upload, when current revision was created, generated by backend", + Type: "RFC 3339", + Example: "2006-01-02T15:04:05.999999999Z07:00", + ReadOnly: true, + }, + "content-type": { + Help: "MIME type, also known as media type", + Type: "string", + Example: "text/plain", + ReadOnly: true, + }, + }, + }, Options: append(oauthutil.SharedOptions, []fs.Option{{ Name: "md5_memory_limit", Help: "Files bigger than this will be cached on disk to calculate the MD5 if required.", @@ -497,7 +524,9 @@ type Object struct { remote string hasMetaData bool size int64 + createTime time.Time modTime time.Time + updateTime time.Time md5 string mimeType string } @@ -972,6 +1001,9 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e CanHaveEmptyDirectories: true, ReadMimeType: true, WriteMimeType: false, + ReadMetadata: true, + WriteMetadata: true, + UserMetadata: false, }).Fill(ctx, f) f.jfsSrv.SetErrorHandler(errorHandler) if opt.TrashedOnly { // we cannot support showing Trashed Files when using ListR right now @@ -1158,6 +1190,7 @@ func parseListRStream(ctx context.Context, r io.Reader, filesystem *Fs, callback remote: filesystem.opt.Enc.ToStandardPath(path.Join(f.Path, f.Name)), size: f.Size, md5: f.Checksum, + createTime: time.Time(f.Created), modTime: time.Time(f.Modified), }) } @@ -1387,7 +1420,7 @@ func (f *Fs) Purge(ctx context.Context, dir string) error { // is currently in trash, but can be made to match, it will be // restored. Returns ErrorObjectNotFound if upload will be necessary // to get a matching remote file. -func (f *Fs) createOrUpdate(ctx context.Context, file string, modTime time.Time, size int64, md5 string) (info *api.JottaFile, err error) { +func (f *Fs) createOrUpdate(ctx context.Context, file string, createTime time.Time, modTime time.Time, size int64, md5 string) (info *api.JottaFile, err error) { opts := rest.Opts{ Method: "POST", Path: f.filePath(file), @@ -1397,11 +1430,10 @@ func (f *Fs) createOrUpdate(ctx context.Context, file string, modTime time.Time, opts.Parameters.Set("cphash", "true") - fileDate := api.JottaTime(modTime).String() opts.ExtraHeaders["JSize"] = strconv.FormatInt(size, 10) opts.ExtraHeaders["JMd5"] = md5 - opts.ExtraHeaders["JCreated"] = fileDate - opts.ExtraHeaders["JModified"] = fileDate + opts.ExtraHeaders["JCreated"] = api.JottaTime(createTime).String() + opts.ExtraHeaders["JModified"] = api.JottaTime(modTime).String() var resp *http.Response err = f.pacer.Call(func() (bool, error) { @@ -1464,7 +1496,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, // if destination was a trashed file then after a successful copy the copied file is still in trash (bug in api?) if err == nil && bool(info.Deleted) && !f.opt.TrashedOnly && info.State == "COMPLETED" { fs.Debugf(src, "Server-side copied to trashed destination, restoring") - info, err = f.createOrUpdate(ctx, remote, srcObj.modTime, srcObj.size, srcObj.md5) + info, err = f.createOrUpdate(ctx, remote, srcObj.createTime, srcObj.modTime, srcObj.size, srcObj.md5) } if err != nil { @@ -1727,7 +1759,9 @@ func (o *Object) setMetaData(info *api.JottaFile) (err error) { o.size = info.Size o.md5 = info.MD5 o.mimeType = info.MimeType + o.createTime = time.Time(info.CreatedAt) o.modTime = time.Time(info.ModifiedAt) + o.updateTime = time.Time(info.UpdatedAt) return nil } @@ -1772,7 +1806,7 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error { // (note that if size/md5 does not match, the file content will // also be modified if deduplication is possible, i.e. it is // important to use correct/latest values) - _, err = o.fs.createOrUpdate(ctx, o.remote, modTime, o.size, o.md5) + _, err = o.fs.createOrUpdate(ctx, o.remote, o.createTime, modTime, o.size, o.md5) if err != nil { if err == fs.ErrorObjectNotFound { // file was modified (size/md5 changed) between readMetaData and createOrUpdate? @@ -1909,6 +1943,37 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op // Wrap the accounting back onto the stream in = wrap(in) } + // Fetch metadata if --metadata is in use + meta, err := fs.GetMetadataOptions(ctx, src, options) + if err != nil { + return fmt.Errorf("failed to read metadata from source object: %w", err) + } + var createdTime string + var modTime string + if meta != nil { + if v, ok := meta["btime"]; ok { + t, err := time.Parse(time.RFC3339Nano, v) // metadata stores RFC3339Nano timestamps + if err != nil { + fs.Debugf(o, "failed to parse metadata btime: %q: %v", v, err) + } else { + createdTime = api.Rfc3339Time(t).String() // jottacloud api wants RFC3339 timestamps + } + } + if v, ok := meta["mtime"]; ok { + t, err := time.Parse(time.RFC3339Nano, v) + if err != nil { + fs.Debugf(o, "failed to parse metadata mtime: %q: %v", v, err) + } else { + modTime = api.Rfc3339Time(t).String() + } + } + } + if modTime == "" { // prefer mtime in meta as Modified time, fallback to source ModTime + modTime = api.Rfc3339Time(src.ModTime(ctx)).String() + } + if createdTime == "" { // if no Created time set same as Modified + createdTime = modTime + } // use the api to allocate the file first and get resume / deduplication info var resp *http.Response @@ -1918,13 +1983,12 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op Options: options, ExtraHeaders: make(map[string]string), } - fileDate := api.Rfc3339Time(src.ModTime(ctx)).String() // the allocate request var request = api.AllocateFileRequest{ Bytes: size, - Created: fileDate, - Modified: fileDate, + Created: createdTime, + Modified: modTime, Md5: md5String, Path: o.fs.allocatePathRaw(o.remote, true), } @@ -1939,7 +2003,10 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op return err } - // If the file state is INCOMPLETE and CORRUPT, try to upload a then + // If the file state is INCOMPLETE and CORRUPT, we must upload it. + // Else, if the file state is COMPLETE, we don't need to upload it because + // the content is already there, possibly it was created with deduplication, + // and also any metadata changes are already performed by the allocate request. if response.State != "COMPLETED" { // how much do we still have to upload? remainingBytes := size - response.ResumePos @@ -1963,22 +2030,18 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op } // send the remaining bytes - resp, err = o.fs.apiSrv.CallJSON(ctx, &opts, nil, &result) + _, err = o.fs.apiSrv.CallJSON(ctx, &opts, nil, &result) if err != nil { return err } - // finally update the meta data - o.hasMetaData = true - o.size = result.Bytes - o.md5 = result.Md5 - o.modTime = time.Unix(result.Modified/1000, 0) - } else { - // If the file state is COMPLETE we don't need to upload it because the file was already found but we still need to update our metadata - return o.readMetaData(ctx, true) + // Upload response contains main metadata properties (size, md5 and modTime) + // which could be set back to the object, but it does not contain the + // necessary information to set the createTime and updateTime properties, + // so must therefore perform a read instead. } - - return nil + // in any case we must update the object meta data + return o.readMetaData(ctx, true) } func (o *Object) remove(ctx context.Context, hard bool) error { @@ -2013,6 +2076,22 @@ func (o *Object) Remove(ctx context.Context) error { return o.remove(ctx, o.fs.opt.HardDelete) } +// Metadata returns metadata for an object +// +// It should return nil if there is no Metadata +func (o *Object) Metadata(ctx context.Context) (metadata fs.Metadata, err error) { + err = o.readMetaData(ctx, false) + if err != nil { + fs.Logf(o, "Failed to read metadata: %v", err) + return nil, err + } + metadata.Set("btime", o.createTime.Format(time.RFC3339Nano)) // metadata timestamps should be RFC3339Nano + metadata.Set("mtime", o.modTime.Format(time.RFC3339Nano)) + metadata.Set("utime", o.updateTime.Format(time.RFC3339Nano)) + metadata.Set("content-type", o.mimeType) + return metadata, nil +} + // Check the interfaces are satisfied var ( _ fs.Fs = (*Fs)(nil) @@ -2027,4 +2106,5 @@ var ( _ fs.CleanUpper = (*Fs)(nil) _ fs.Object = (*Object)(nil) _ fs.MimeTyper = (*Object)(nil) + _ fs.Metadataer = (*Object)(nil) ) diff --git a/backend/jottacloud/jottacloud_internal_test.go b/backend/jottacloud/jottacloud_internal_test.go index b2b11758a..f77a3f291 100644 --- a/backend/jottacloud/jottacloud_internal_test.go +++ b/backend/jottacloud/jottacloud_internal_test.go @@ -1,11 +1,17 @@ package jottacloud import ( + "context" "crypto/md5" "fmt" "io" "testing" + "time" + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fstest" + "github.com/rclone/rclone/fstest/fstests" + "github.com/rclone/rclone/lib/random" "github.com/rclone/rclone/lib/readers" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -40,3 +46,48 @@ func TestReadMD5(t *testing.T) { }) } } + +func (f *Fs) InternalTestMetadata(t *testing.T) { + ctx := context.Background() + contents := random.String(1000) + + item := fstest.NewItem("test-metadata", contents, fstest.Time("2001-05-06T04:05:06.499999999Z")) + utime := time.Now() + metadata := fs.Metadata{ + "btime": "2009-05-06T04:05:06.499999999Z", + "mtime": "2010-06-07T08:09:07.599999999Z", + //"utime" - read-only + //"content-type" - read-only + } + obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, contents, true, "text/html", metadata) + defer func() { + assert.NoError(t, obj.Remove(ctx)) + }() + o := obj.(*Object) + gotMetadata, err := o.Metadata(ctx) + require.NoError(t, err) + for k, v := range metadata { + got := gotMetadata[k] + switch k { + case "btime": + assert.True(t, fstest.Time(v).Truncate(f.Precision()).Equal(fstest.Time(got)), fmt.Sprintf("btime not equal want %v got %v", v, got)) + case "mtime": + assert.True(t, fstest.Time(v).Truncate(f.Precision()).Equal(fstest.Time(got)), fmt.Sprintf("btime not equal want %v got %v", v, got)) + case "utime": + gotUtime := fstest.Time(got) + dt := gotUtime.Sub(utime) + assert.True(t, dt < time.Minute && dt > -time.Minute, fmt.Sprintf("utime more than 1 minute out want %v got %v delta %v", utime, gotUtime, dt)) + assert.True(t, fstest.Time(v).Equal(fstest.Time(got))) + case "content-type": + assert.True(t, o.MimeType(ctx) == got) + default: + assert.Equal(t, v, got, k) + } + } +} + +func (f *Fs) InternalTest(t *testing.T) { + t.Run("Metadata", f.InternalTestMetadata) +} + +var _ fstests.InternalTester = (*Fs)(nil) diff --git a/docs/content/jottacloud.md b/docs/content/jottacloud.md index 918b499f6..d2b19e53e 100644 --- a/docs/content/jottacloud.md +++ b/docs/content/jottacloud.md @@ -233,7 +233,7 @@ them. Generally you should avoid these, unless you know what you are doing. ### --fast-list -This remote supports `--fast-list` which allows you to use fewer +This backend supports `--fast-list` which allows you to use fewer transactions in exchange for more memory. See the [rclone docs](/docs/#fast-list) for more details. @@ -241,8 +241,9 @@ Note that the implementation in Jottacloud always uses only a single API request to get the entire list, so for large folders this could lead to long wait time before the first results are shown. -Note also that with rclone version 1.58 and newer information about -[MIME types](/overview/#mime-type) are not available when using `--fast-list`. +Note also that with rclone version 1.58 and newer, information about +[MIME types](/overview/#mime-type) and metadata item [utime](#metadata) +are not available when using `--fast-list`. ### Modified time and hashes diff --git a/docs/content/overview.md b/docs/content/overview.md index a0759e579..37e10c589 100644 --- a/docs/content/overview.md +++ b/docs/content/overview.md @@ -33,7 +33,7 @@ Here is an overview of the major features of each cloud storage system. | HiDrive | HiDrive ¹² | R/W | No | No | - | - | | HTTP | - | R | No | No | R | - | | Internet Archive | MD5, SHA1, CRC32 | R/W ¹¹ | No | No | - | RWU | -| Jottacloud | MD5 | R/W | Yes | No | R | - | +| Jottacloud | MD5 | R/W | Yes | No | R | RW | | Koofr | MD5 | - | Yes | No | - | - | | Mail.ru Cloud | Mailru ⁶ | R/W | Yes | No | - | - | | Mega | - | - | No | Yes | - | - |