drive: add support of SHA-1 and SHA-256 checksum
This commit is contained in:
rinsuki
Nick Craig-Wood
parent
9e80d48b03
commit
8fd66daab6
|
|
@ -71,7 +71,7 @@ const (
|
||||||
// 1<<18 is the minimum size supported by the Google uploader, and there is no maximum.
|
// 1<<18 is the minimum size supported by the Google uploader, and there is no maximum.
|
||||||
minChunkSize = fs.SizeSuffix(googleapi.MinUploadChunkSize)
|
minChunkSize = fs.SizeSuffix(googleapi.MinUploadChunkSize)
|
||||||
defaultChunkSize = 8 * fs.Mebi
|
defaultChunkSize = 8 * fs.Mebi
|
||||||
partialFields = "id,name,size,md5Checksum,trashed,explicitlyTrashed,modifiedTime,createdTime,mimeType,parents,webViewLink,shortcutDetails,exportLinks,resourceKey"
|
partialFields = "id,name,size,md5Checksum,sha1Checksum,sha256Checksum,trashed,explicitlyTrashed,modifiedTime,createdTime,mimeType,parents,webViewLink,shortcutDetails,exportLinks,resourceKey"
|
||||||
listRGrouping = 50 // number of IDs to search at once when using ListR
|
listRGrouping = 50 // number of IDs to search at once when using ListR
|
||||||
listRInputBuffer = 1000 // size of input buffer when using ListR
|
listRInputBuffer = 1000 // size of input buffer when using ListR
|
||||||
defaultXDGIcon = "text-html"
|
defaultXDGIcon = "text-html"
|
||||||
|
|
@ -323,13 +323,13 @@ rather than shortcuts themselves when doing server side copies.`,
|
||||||
}, {
|
}, {
|
||||||
Name: "skip_checksum_gphotos",
|
Name: "skip_checksum_gphotos",
|
||||||
Default: false,
|
Default: false,
|
||||||
Help: `Skip MD5 checksum on Google photos and videos only.
|
Help: `Skip checksums on Google photos and videos only.
|
||||||
|
|
||||||
Use this if you get checksum errors when transferring Google photos or
|
Use this if you get checksum errors when transferring Google photos or
|
||||||
videos.
|
videos.
|
||||||
|
|
||||||
Setting this flag will cause Google photos and videos to return a
|
Setting this flag will cause Google photos and videos to return a
|
||||||
blank MD5 checksum.
|
blank checksums.
|
||||||
|
|
||||||
Google photos are identified by being in the "photos" space.
|
Google photos are identified by being in the "photos" space.
|
||||||
|
|
||||||
|
|
@ -751,6 +751,8 @@ type Object struct {
|
||||||
baseObject
|
baseObject
|
||||||
url string // Download URL of this object
|
url string // Download URL of this object
|
||||||
md5sum string // md5sum of the object
|
md5sum string // md5sum of the object
|
||||||
|
sha1sum string // sha1sum of the object
|
||||||
|
sha256sum string // sha256sum of the object
|
||||||
v2Download bool // generate v2 download link ondemand
|
v2Download bool // generate v2 download link ondemand
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -1414,6 +1416,8 @@ func (f *Fs) newRegularObject(remote string, info *drive.File) fs.Object {
|
||||||
for _, space := range info.Spaces {
|
for _, space := range info.Spaces {
|
||||||
if space == "photos" {
|
if space == "photos" {
|
||||||
info.Md5Checksum = ""
|
info.Md5Checksum = ""
|
||||||
|
info.Sha1Checksum = ""
|
||||||
|
info.Sha256Checksum = ""
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -1422,6 +1426,8 @@ func (f *Fs) newRegularObject(remote string, info *drive.File) fs.Object {
|
||||||
baseObject: f.newBaseObject(remote, info),
|
baseObject: f.newBaseObject(remote, info),
|
||||||
url: fmt.Sprintf("%sfiles/%s?alt=media", f.svc.BasePath, actualID(info.Id)),
|
url: fmt.Sprintf("%sfiles/%s?alt=media", f.svc.BasePath, actualID(info.Id)),
|
||||||
md5sum: strings.ToLower(info.Md5Checksum),
|
md5sum: strings.ToLower(info.Md5Checksum),
|
||||||
|
sha1sum: strings.ToLower(info.Sha1Checksum),
|
||||||
|
sha256sum: strings.ToLower(info.Sha256Checksum),
|
||||||
v2Download: f.opt.V2DownloadMinSize != -1 && info.Size >= int64(f.opt.V2DownloadMinSize),
|
v2Download: f.opt.V2DownloadMinSize != -1 && info.Size >= int64(f.opt.V2DownloadMinSize),
|
||||||
}
|
}
|
||||||
if info.ResourceKey != "" {
|
if info.ResourceKey != "" {
|
||||||
|
|
@ -3007,7 +3013,7 @@ func (f *Fs) DirCacheFlush() {
|
||||||
|
|
||||||
// Hashes returns the supported hash sets.
|
// Hashes returns the supported hash sets.
|
||||||
func (f *Fs) Hashes() hash.Set {
|
func (f *Fs) Hashes() hash.Set {
|
||||||
return hash.Set(hash.MD5)
|
return hash.NewHashSet(hash.MD5, hash.SHA1, hash.SHA256)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *Fs) changeChunkSize(chunkSizeString string) (err error) {
|
func (f *Fs) changeChunkSize(chunkSizeString string) (err error) {
|
||||||
|
|
@ -3568,10 +3574,16 @@ func (o *baseObject) Remote() string {
|
||||||
|
|
||||||
// Hash returns the Md5sum of an object returning a lowercase hex string
|
// Hash returns the Md5sum of an object returning a lowercase hex string
|
||||||
func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
|
func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
|
||||||
if t != hash.MD5 {
|
if t == hash.MD5 {
|
||||||
return "", hash.ErrUnsupported
|
return o.md5sum, nil
|
||||||
}
|
}
|
||||||
return o.md5sum, nil
|
if t == hash.SHA1 {
|
||||||
|
return o.sha1sum, nil
|
||||||
|
}
|
||||||
|
if t == hash.SHA256 {
|
||||||
|
return o.sha256sum, nil
|
||||||
|
}
|
||||||
|
return "", hash.ErrUnsupported
|
||||||
}
|
}
|
||||||
func (o *baseObject) Hash(ctx context.Context, t hash.Type) (string, error) {
|
func (o *baseObject) Hash(ctx context.Context, t hash.Type) (string, error) {
|
||||||
if t != hash.MD5 {
|
if t != hash.MD5 {
|
||||||
|
|
|
||||||
|
|
@ -772,13 +772,13 @@ Properties:
|
||||||
|
|
||||||
#### --drive-skip-checksum-gphotos
|
#### --drive-skip-checksum-gphotos
|
||||||
|
|
||||||
Skip MD5 checksum on Google photos and videos only.
|
Skip checksums on Google photos and videos only.
|
||||||
|
|
||||||
Use this if you get checksum errors when transferring Google photos or
|
Use this if you get checksum errors when transferring Google photos or
|
||||||
videos.
|
videos.
|
||||||
|
|
||||||
Setting this flag will cause Google photos and videos to return a
|
Setting this flag will cause Google photos and videos to return a
|
||||||
blank MD5 checksum.
|
blank checksums.
|
||||||
|
|
||||||
Google photos are identified by being in the "photos" space.
|
Google photos are identified by being in the "photos" space.
|
||||||
|
|
||||||
|
|
@ -1526,6 +1526,10 @@ Waiting a moderate period of time between attempts (estimated to be
|
||||||
approximately 1 hour) and/or not using --fast-list both seem to be
|
approximately 1 hour) and/or not using --fast-list both seem to be
|
||||||
effective in preventing the problem.
|
effective in preventing the problem.
|
||||||
|
|
||||||
|
### Hashes
|
||||||
|
|
||||||
|
We need to say that all files have MD5 hashes, but a small fraction of files uploaded may not have SHA1 or SHA256 hashes especially if they were uploaded before 2018.
|
||||||
|
|
||||||
## Making your own client_id
|
## Making your own client_id
|
||||||
|
|
||||||
When you use rclone with Google drive in its default configuration you
|
When you use rclone with Google drive in its default configuration you
|
||||||
|
|
|
||||||
|
|
@ -14,53 +14,53 @@ show through.
|
||||||
|
|
||||||
Here is an overview of the major features of each cloud storage system.
|
Here is an overview of the major features of each cloud storage system.
|
||||||
|
|
||||||
| Name | Hash | ModTime | Case Insensitive | Duplicate Files | MIME Type | Metadata |
|
| Name | Hash | ModTime | Case Insensitive | Duplicate Files | MIME Type | Metadata |
|
||||||
| ---------------------------- |:----------------:|:-------:|:----------------:|:---------------:|:---------:|:--------:|
|
| ---------------------------- |:-----------------:|:-------:|:----------------:|:---------------:|:---------:|:--------:|
|
||||||
| 1Fichier | Whirlpool | - | No | Yes | R | - |
|
| 1Fichier | Whirlpool | - | No | Yes | R | - |
|
||||||
| Akamai Netstorage | MD5, SHA256 | R/W | No | No | R | - |
|
| Akamai Netstorage | MD5, SHA256 | R/W | No | No | R | - |
|
||||||
| Amazon Drive | MD5 | - | Yes | No | R | - |
|
| Amazon Drive | MD5 | - | Yes | No | R | - |
|
||||||
| Amazon S3 (or S3 compatible) | MD5 | R/W | No | No | R/W | RWU |
|
| Amazon S3 (or S3 compatible) | MD5 | R/W | No | No | R/W | RWU |
|
||||||
| Backblaze B2 | SHA1 | R/W | No | No | R/W | - |
|
| Backblaze B2 | SHA1 | R/W | No | No | R/W | - |
|
||||||
| Box | SHA1 | R/W | Yes | No | - | - |
|
| Box | SHA1 | R/W | Yes | No | - | - |
|
||||||
| Citrix ShareFile | MD5 | R/W | Yes | No | - | - |
|
| Citrix ShareFile | MD5 | R/W | Yes | No | - | - |
|
||||||
| Dropbox | DBHASH ¹ | R | Yes | No | - | - |
|
| Dropbox | DBHASH ¹ | R | Yes | No | - | - |
|
||||||
| Enterprise File Fabric | - | R/W | Yes | No | R/W | - |
|
| Enterprise File Fabric | - | R/W | Yes | No | R/W | - |
|
||||||
| FTP | - | R/W ¹⁰ | No | No | - | - |
|
| FTP | - | R/W ¹⁰ | No | No | - | - |
|
||||||
| Google Cloud Storage | MD5 | R/W | No | No | R/W | - |
|
| Google Cloud Storage | MD5 | R/W | No | No | R/W | - |
|
||||||
| Google Drive | MD5 | R/W | No | Yes | R/W | - |
|
| Google Drive | MD5, SHA1, SHA256 | R/W | No | Yes | R/W | - |
|
||||||
| Google Photos | - | - | No | Yes | R | - |
|
| Google Photos | - | - | No | Yes | R | - |
|
||||||
| HDFS | - | R/W | No | No | - | - |
|
| HDFS | - | R/W | No | No | - | - |
|
||||||
| HiDrive | HiDrive ¹² | R/W | No | No | - | - |
|
| HiDrive | HiDrive ¹² | R/W | No | No | - | - |
|
||||||
| HTTP | - | R | No | No | R | - |
|
| HTTP | - | R | No | No | R | - |
|
||||||
| Internet Archive | MD5, SHA1, CRC32 | R/W ¹¹ | No | No | - | RWU |
|
| Internet Archive | MD5, SHA1, CRC32 | R/W ¹¹ | No | No | - | RWU |
|
||||||
| Jottacloud | MD5 | R/W | Yes | No | R | - |
|
| Jottacloud | MD5 | R/W | Yes | No | R | - |
|
||||||
| Koofr | MD5 | - | Yes | No | - | - |
|
| Koofr | MD5 | - | Yes | No | - | - |
|
||||||
| Mail.ru Cloud | Mailru ⁶ | R/W | Yes | No | - | - |
|
| Mail.ru Cloud | Mailru ⁶ | R/W | Yes | No | - | - |
|
||||||
| Mega | - | - | No | Yes | - | - |
|
| Mega | - | - | No | Yes | - | - |
|
||||||
| Memory | MD5 | R/W | No | No | - | - |
|
| Memory | MD5 | R/W | No | No | - | - |
|
||||||
| Microsoft Azure Blob Storage | MD5 | R/W | No | No | R/W | - |
|
| Microsoft Azure Blob Storage | MD5 | R/W | No | No | R/W | - |
|
||||||
| Microsoft OneDrive | QuickXorHash ⁵ | R/W | Yes | No | R | - |
|
| Microsoft OneDrive | QuickXorHash ⁵ | R/W | Yes | No | R | - |
|
||||||
| OpenDrive | MD5 | R/W | Yes | Partial ⁸ | - | - |
|
| OpenDrive | MD5 | R/W | Yes | Partial ⁸ | - | - |
|
||||||
| OpenStack Swift | MD5 | R/W | No | No | R/W | - |
|
| OpenStack Swift | MD5 | R/W | No | No | R/W | - |
|
||||||
| Oracle Object Storage | MD5 | R/W | No | No | R/W | - |
|
| Oracle Object Storage | MD5 | R/W | No | No | R/W | - |
|
||||||
| pCloud | MD5, SHA1 ⁷ | R | No | No | W | - |
|
| pCloud | MD5, SHA1 ⁷ | R | No | No | W | - |
|
||||||
| PikPak | MD5 | R | No | No | R | - |
|
| PikPak | MD5 | R | No | No | R | - |
|
||||||
| premiumize.me | - | - | Yes | No | R | - |
|
| premiumize.me | - | - | Yes | No | R | - |
|
||||||
| put.io | CRC-32 | R/W | No | Yes | R | - |
|
| put.io | CRC-32 | R/W | No | Yes | R | - |
|
||||||
| Proton Drive | SHA1 | R/W | No | No | R | - |
|
| Proton Drive | SHA1 | R/W | No | No | R | - |
|
||||||
| QingStor | MD5 | - ⁹ | No | No | R/W | - |
|
| QingStor | MD5 | - ⁹ | No | No | R/W | - |
|
||||||
| Quatrix by Maytech | - | R/W | No | No | - | - |
|
| Quatrix by Maytech | - | R/W | No | No | - | - |
|
||||||
| Seafile | - | - | No | No | - | - |
|
| Seafile | - | - | No | No | - | - |
|
||||||
| SFTP | MD5, SHA1 ² | R/W | Depends | No | - | - |
|
| SFTP | MD5, SHA1 ² | R/W | Depends | No | - | - |
|
||||||
| Sia | - | - | No | No | - | - |
|
| Sia | - | - | No | No | - | - |
|
||||||
| SMB | - | - | Yes | No | - | - |
|
| SMB | - | - | Yes | No | - | - |
|
||||||
| SugarSync | - | - | No | No | - | - |
|
| SugarSync | - | - | No | No | - | - |
|
||||||
| Storj | - | R | No | No | - | - |
|
| Storj | - | R | No | No | - | - |
|
||||||
| Uptobox | - | - | No | Yes | - | - |
|
| Uptobox | - | - | No | Yes | - | - |
|
||||||
| WebDAV | MD5, SHA1 ³ | R ⁴ | Depends | No | - | - |
|
| WebDAV | MD5, SHA1 ³ | R ⁴ | Depends | No | - | - |
|
||||||
| Yandex Disk | MD5 | R/W | No | No | R | - |
|
| Yandex Disk | MD5 | R/W | No | No | R | - |
|
||||||
| Zoho WorkDrive | - | - | No | No | - | - |
|
| Zoho WorkDrive | - | - | No | No | - | - |
|
||||||
| The local filesystem | All | R/W | Depends | No | - | RWU |
|
| The local filesystem | All | R/W | Depends | No | - | RWU |
|
||||||
|
|
||||||
### Notes
|
### Notes
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue