diff --git a/backend/internetarchive/internetarchive.go b/backend/internetarchive/internetarchive.go index 9d4dd36db..b7a0676b5 100644 --- a/backend/internetarchive/internetarchive.go +++ b/backend/internetarchive/internetarchive.go @@ -42,54 +42,70 @@ func init() { MetadataInfo: &fs.MetadataInfo{ System: map[string]fs.MetadataHelp{ "name": { - Help: "Full file path, without the bucket part", - Type: "filename", - Example: "backend/internetarchive/internetarchive.go", + Help: "Full file path, without the bucket part", + Type: "filename", + Example: "backend/internetarchive/internetarchive.go", + ReadOnly: true, }, "source": { - Help: "The source of the file", - Type: "string", - Example: "original", + Help: "The source of the file", + Type: "string", + Example: "original", + ReadOnly: true, }, "mtime": { - Help: "Time of last modification, managed by Rclone", - Type: "RFC 3339", - Example: "2006-01-02T15:04:05.999999999Z", + Help: "Time of last modification, managed by Rclone", + Type: "RFC 3339", + Example: "2006-01-02T15:04:05.999999999Z", + ReadOnly: true, }, "size": { - Help: "File size in bytes", - Type: "decimal number", - Example: "123456", + Help: "File size in bytes", + Type: "decimal number", + Example: "123456", + ReadOnly: true, }, "md5": { - Help: "MD5 hash calculated by Internet Archive", - Type: "string", - Example: "01234567012345670123456701234567", + Help: "MD5 hash calculated by Internet Archive", + Type: "string", + Example: "01234567012345670123456701234567", + ReadOnly: true, }, "crc32": { - Help: "CRC32 calculated by Internet Archive", - Type: "string", - Example: "01234567", + Help: "CRC32 calculated by Internet Archive", + Type: "string", + Example: "01234567", + ReadOnly: true, }, "sha1": { - Help: "SHA1 hash calculated by Internet Archive", - Type: "string", - Example: "0123456701234567012345670123456701234567", + Help: "SHA1 hash calculated by Internet Archive", + Type: "string", + Example: "0123456701234567012345670123456701234567", + ReadOnly: true, }, "format": { - Help: "Name of format identified by Internet Archive", - Type: "string", - Example: "Comma-Separated Values", + Help: "Name of format identified by Internet Archive", + Type: "string", + Example: "Comma-Separated Values", + ReadOnly: true, }, "old_version": { - Help: "Whether the file was replaced and moved by keep-old-version flag", - Type: "boolean", - Example: "true", + Help: "Whether the file was replaced and moved by keep-old-version flag", + Type: "boolean", + Example: "true", + ReadOnly: true, }, "viruscheck": { - Help: "The last time viruscheck process was run for the file (?)", - Type: "unixtime", - Example: "1654191352", + Help: "The last time viruscheck process was run for the file (?)", + Type: "unixtime", + Example: "1654191352", + ReadOnly: true, + }, + "summation": { + Help: "Check https://forum.rclone.org/t/31922 for how it is used", + Type: "string", + Example: "md5", + ReadOnly: true, }, "rclone-ia-mtime": { @@ -173,7 +189,7 @@ var roMetadataKey = map[string]interface{}{ // do not add mtime here, it's a documented exception "name": nil, "source": nil, "size": nil, "md5": nil, "crc32": nil, "sha1": nil, "format": nil, "old_version": nil, - "viruscheck": nil, + "viruscheck": nil, "summation": nil, } // Options defines the configuration for this backend @@ -222,6 +238,7 @@ type IAFile struct { Md5 string `json:"md5"` Crc32 string `json:"crc32"` Sha1 string `json:"sha1"` + Summation string `json:"summation"` rawData json.RawMessage } @@ -1135,16 +1152,21 @@ func (f *Fs) waitDelete(ctx context.Context, bucket, bucketPath string) (err err } func makeValidObject(f *Fs, remote string, file IAFile, mtime time.Time, size int64) *Object { - return &Object{ + ret := &Object{ fs: f, remote: remote, modTime: mtime, size: size, - md5: file.Md5, - crc32: file.Crc32, - sha1: file.Sha1, rawData: file.rawData, } + // hashes from _files.xml (where summation != "") is different from one in other files + // https://forum.rclone.org/t/internet-archive-md5-tag-in-id-files-xml-interpreted-incorrectly/31922 + if file.Summation == "" { + ret.md5 = file.Md5 + ret.crc32 = file.Crc32 + ret.sha1 = file.Sha1 + } + return ret } func makeValidObject2(f *Fs, file IAFile, bucket string) *Object { diff --git a/docs/content/internetarchive.md b/docs/content/internetarchive.md index 1bdb05962..ae3393d80 100644 --- a/docs/content/internetarchive.md +++ b/docs/content/internetarchive.md @@ -12,11 +12,10 @@ Refer to [IAS3 API documentation](https://archive.org/services/docs/api/ias3.htm Paths are specified as `remote:bucket` (or `remote:` for the `lsd` command.) You may put subdirectories in too, e.g. `remote:item/path/to/dir`. -Once you have made a remote (see the provider specific section above) -you can use it like this: - Unlike S3, listing up all items uploaded by you isn't supported. +Once you have made a remote, you can use it like this: + Make a new item rclone mkdir remote:item @@ -53,6 +52,7 @@ The following are reserved by Internet Archive: - `format` - `old_version` - `viruscheck` +- `summation` Trying to set values to these keys is ignored with a warning. Only setting `mtime` is an exception. Doing so make it the identical behavior as setting ModTime. @@ -258,19 +258,20 @@ Here are the possible system metadata items for the internetarchive backend. | Name | Help | Type | Example | Read Only | |------|------|------|---------|-----------| -| crc32 | CRC32 calculated by Internet Archive | string | 01234567 | N | -| format | Name of format identified by Internet Archive | string | Comma-Separated Values | N | -| md5 | MD5 hash calculated by Internet Archive | string | 01234567012345670123456701234567 | N | -| mtime | Time of last modification, managed by Rclone | RFC 3339 | 2006-01-02T15:04:05.999999999Z | N | -| name | Full file path, without the bucket part | filename | backend/internetarchive/internetarchive.go | N | -| old_version | Whether the file was replaced and moved by keep-old-version flag | boolean | true | N | +| crc32 | CRC32 calculated by Internet Archive | string | 01234567 | **Y** | +| format | Name of format identified by Internet Archive | string | Comma-Separated Values | **Y** | +| md5 | MD5 hash calculated by Internet Archive | string | 01234567012345670123456701234567 | **Y** | +| mtime | Time of last modification, managed by Rclone | RFC 3339 | 2006-01-02T15:04:05.999999999Z | **Y** | +| name | Full file path, without the bucket part | filename | backend/internetarchive/internetarchive.go | **Y** | +| old_version | Whether the file was replaced and moved by keep-old-version flag | boolean | true | **Y** | | rclone-ia-mtime | Time of last modification, managed by Internet Archive | RFC 3339 | 2006-01-02T15:04:05.999999999Z | N | | rclone-mtime | Time of last modification, managed by Rclone | RFC 3339 | 2006-01-02T15:04:05.999999999Z | N | | rclone-update-track | Random value used by Rclone for tracking changes inside Internet Archive | string | aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa | N | -| sha1 | SHA1 hash calculated by Internet Archive | string | 0123456701234567012345670123456701234567 | N | -| size | File size in bytes | decimal number | 123456 | N | -| source | The source of the file | string | original | N | -| viruscheck | The last time viruscheck process was run for the file (?) | unixtime | 1654191352 | N | +| sha1 | SHA1 hash calculated by Internet Archive | string | 0123456701234567012345670123456701234567 | **Y** | +| size | File size in bytes | decimal number | 123456 | **Y** | +| source | The source of the file | string | original | **Y** | +| summation | Check https://forum.rclone.org/t/31922 for how it is used | string | md5 | **Y** | +| viruscheck | The last time viruscheck process was run for the file (?) | unixtime | 1654191352 | **Y** | See the [metadata](/docs/#metadata) docs for more info.