backend/internetarchive: ignore checksums for files using the different method
This commit is contained in:
Lesmiscore
Nick Craig-Wood
parent
74401077dc
commit
37bcc3df14
|
|
@ -42,54 +42,70 @@ func init() {
|
|||
MetadataInfo: &fs.MetadataInfo{
|
||||
System: map[string]fs.MetadataHelp{
|
||||
"name": {
|
||||
Help: "Full file path, without the bucket part",
|
||||
Type: "filename",
|
||||
Example: "backend/internetarchive/internetarchive.go",
|
||||
Help: "Full file path, without the bucket part",
|
||||
Type: "filename",
|
||||
Example: "backend/internetarchive/internetarchive.go",
|
||||
ReadOnly: true,
|
||||
},
|
||||
"source": {
|
||||
Help: "The source of the file",
|
||||
Type: "string",
|
||||
Example: "original",
|
||||
Help: "The source of the file",
|
||||
Type: "string",
|
||||
Example: "original",
|
||||
ReadOnly: true,
|
||||
},
|
||||
"mtime": {
|
||||
Help: "Time of last modification, managed by Rclone",
|
||||
Type: "RFC 3339",
|
||||
Example: "2006-01-02T15:04:05.999999999Z",
|
||||
Help: "Time of last modification, managed by Rclone",
|
||||
Type: "RFC 3339",
|
||||
Example: "2006-01-02T15:04:05.999999999Z",
|
||||
ReadOnly: true,
|
||||
},
|
||||
"size": {
|
||||
Help: "File size in bytes",
|
||||
Type: "decimal number",
|
||||
Example: "123456",
|
||||
Help: "File size in bytes",
|
||||
Type: "decimal number",
|
||||
Example: "123456",
|
||||
ReadOnly: true,
|
||||
},
|
||||
"md5": {
|
||||
Help: "MD5 hash calculated by Internet Archive",
|
||||
Type: "string",
|
||||
Example: "01234567012345670123456701234567",
|
||||
Help: "MD5 hash calculated by Internet Archive",
|
||||
Type: "string",
|
||||
Example: "01234567012345670123456701234567",
|
||||
ReadOnly: true,
|
||||
},
|
||||
"crc32": {
|
||||
Help: "CRC32 calculated by Internet Archive",
|
||||
Type: "string",
|
||||
Example: "01234567",
|
||||
Help: "CRC32 calculated by Internet Archive",
|
||||
Type: "string",
|
||||
Example: "01234567",
|
||||
ReadOnly: true,
|
||||
},
|
||||
"sha1": {
|
||||
Help: "SHA1 hash calculated by Internet Archive",
|
||||
Type: "string",
|
||||
Example: "0123456701234567012345670123456701234567",
|
||||
Help: "SHA1 hash calculated by Internet Archive",
|
||||
Type: "string",
|
||||
Example: "0123456701234567012345670123456701234567",
|
||||
ReadOnly: true,
|
||||
},
|
||||
"format": {
|
||||
Help: "Name of format identified by Internet Archive",
|
||||
Type: "string",
|
||||
Example: "Comma-Separated Values",
|
||||
Help: "Name of format identified by Internet Archive",
|
||||
Type: "string",
|
||||
Example: "Comma-Separated Values",
|
||||
ReadOnly: true,
|
||||
},
|
||||
"old_version": {
|
||||
Help: "Whether the file was replaced and moved by keep-old-version flag",
|
||||
Type: "boolean",
|
||||
Example: "true",
|
||||
Help: "Whether the file was replaced and moved by keep-old-version flag",
|
||||
Type: "boolean",
|
||||
Example: "true",
|
||||
ReadOnly: true,
|
||||
},
|
||||
"viruscheck": {
|
||||
Help: "The last time viruscheck process was run for the file (?)",
|
||||
Type: "unixtime",
|
||||
Example: "1654191352",
|
||||
Help: "The last time viruscheck process was run for the file (?)",
|
||||
Type: "unixtime",
|
||||
Example: "1654191352",
|
||||
ReadOnly: true,
|
||||
},
|
||||
"summation": {
|
||||
Help: "Check https://forum.rclone.org/t/31922 for how it is used",
|
||||
Type: "string",
|
||||
Example: "md5",
|
||||
ReadOnly: true,
|
||||
},
|
||||
|
||||
"rclone-ia-mtime": {
|
||||
|
|
@ -173,7 +189,7 @@ var roMetadataKey = map[string]interface{}{
|
|||
// do not add mtime here, it's a documented exception
|
||||
"name": nil, "source": nil, "size": nil, "md5": nil,
|
||||
"crc32": nil, "sha1": nil, "format": nil, "old_version": nil,
|
||||
"viruscheck": nil,
|
||||
"viruscheck": nil, "summation": nil,
|
||||
}
|
||||
|
||||
// Options defines the configuration for this backend
|
||||
|
|
@ -222,6 +238,7 @@ type IAFile struct {
|
|||
Md5 string `json:"md5"`
|
||||
Crc32 string `json:"crc32"`
|
||||
Sha1 string `json:"sha1"`
|
||||
Summation string `json:"summation"`
|
||||
|
||||
rawData json.RawMessage
|
||||
}
|
||||
|
|
@ -1135,16 +1152,21 @@ func (f *Fs) waitDelete(ctx context.Context, bucket, bucketPath string) (err err
|
|||
}
|
||||
|
||||
func makeValidObject(f *Fs, remote string, file IAFile, mtime time.Time, size int64) *Object {
|
||||
return &Object{
|
||||
ret := &Object{
|
||||
fs: f,
|
||||
remote: remote,
|
||||
modTime: mtime,
|
||||
size: size,
|
||||
md5: file.Md5,
|
||||
crc32: file.Crc32,
|
||||
sha1: file.Sha1,
|
||||
rawData: file.rawData,
|
||||
}
|
||||
// hashes from _files.xml (where summation != "") is different from one in other files
|
||||
// https://forum.rclone.org/t/internet-archive-md5-tag-in-id-files-xml-interpreted-incorrectly/31922
|
||||
if file.Summation == "" {
|
||||
ret.md5 = file.Md5
|
||||
ret.crc32 = file.Crc32
|
||||
ret.sha1 = file.Sha1
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
func makeValidObject2(f *Fs, file IAFile, bucket string) *Object {
|
||||
|
|
|
|||
|
|
@ -12,11 +12,10 @@ Refer to [IAS3 API documentation](https://archive.org/services/docs/api/ias3.htm
|
|||
Paths are specified as `remote:bucket` (or `remote:` for the `lsd`
|
||||
command.) You may put subdirectories in too, e.g. `remote:item/path/to/dir`.
|
||||
|
||||
Once you have made a remote (see the provider specific section above)
|
||||
you can use it like this:
|
||||
|
||||
Unlike S3, listing up all items uploaded by you isn't supported.
|
||||
|
||||
Once you have made a remote, you can use it like this:
|
||||
|
||||
Make a new item
|
||||
|
||||
rclone mkdir remote:item
|
||||
|
|
@ -53,6 +52,7 @@ The following are reserved by Internet Archive:
|
|||
- `format`
|
||||
- `old_version`
|
||||
- `viruscheck`
|
||||
- `summation`
|
||||
|
||||
Trying to set values to these keys is ignored with a warning.
|
||||
Only setting `mtime` is an exception. Doing so make it the identical behavior as setting ModTime.
|
||||
|
|
@ -258,19 +258,20 @@ Here are the possible system metadata items for the internetarchive backend.
|
|||
|
||||
| Name | Help | Type | Example | Read Only |
|
||||
|------|------|------|---------|-----------|
|
||||
| crc32 | CRC32 calculated by Internet Archive | string | 01234567 | N |
|
||||
| format | Name of format identified by Internet Archive | string | Comma-Separated Values | N |
|
||||
| md5 | MD5 hash calculated by Internet Archive | string | 01234567012345670123456701234567 | N |
|
||||
| mtime | Time of last modification, managed by Rclone | RFC 3339 | 2006-01-02T15:04:05.999999999Z | N |
|
||||
| name | Full file path, without the bucket part | filename | backend/internetarchive/internetarchive.go | N |
|
||||
| old_version | Whether the file was replaced and moved by keep-old-version flag | boolean | true | N |
|
||||
| crc32 | CRC32 calculated by Internet Archive | string | 01234567 | **Y** |
|
||||
| format | Name of format identified by Internet Archive | string | Comma-Separated Values | **Y** |
|
||||
| md5 | MD5 hash calculated by Internet Archive | string | 01234567012345670123456701234567 | **Y** |
|
||||
| mtime | Time of last modification, managed by Rclone | RFC 3339 | 2006-01-02T15:04:05.999999999Z | **Y** |
|
||||
| name | Full file path, without the bucket part | filename | backend/internetarchive/internetarchive.go | **Y** |
|
||||
| old_version | Whether the file was replaced and moved by keep-old-version flag | boolean | true | **Y** |
|
||||
| rclone-ia-mtime | Time of last modification, managed by Internet Archive | RFC 3339 | 2006-01-02T15:04:05.999999999Z | N |
|
||||
| rclone-mtime | Time of last modification, managed by Rclone | RFC 3339 | 2006-01-02T15:04:05.999999999Z | N |
|
||||
| rclone-update-track | Random value used by Rclone for tracking changes inside Internet Archive | string | aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa | N |
|
||||
| sha1 | SHA1 hash calculated by Internet Archive | string | 0123456701234567012345670123456701234567 | N |
|
||||
| size | File size in bytes | decimal number | 123456 | N |
|
||||
| source | The source of the file | string | original | N |
|
||||
| viruscheck | The last time viruscheck process was run for the file (?) | unixtime | 1654191352 | N |
|
||||
| sha1 | SHA1 hash calculated by Internet Archive | string | 0123456701234567012345670123456701234567 | **Y** |
|
||||
| size | File size in bytes | decimal number | 123456 | **Y** |
|
||||
| source | The source of the file | string | original | **Y** |
|
||||
| summation | Check https://forum.rclone.org/t/31922 for how it is used | string | md5 | **Y** |
|
||||
| viruscheck | The last time viruscheck process was run for the file (?) | unixtime | 1654191352 | **Y** |
|
||||
|
||||
See the [metadata](/docs/#metadata) docs for more info.
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue