community
/
rclone
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

onedrive: add QuickXorHash support for OneDrive for business - fixes #2262

This commit is contained in:
Nick Craig-Wood 2018-04-20 12:55:49 +01:00
parent bcdb7719c6
commit 1a3fb21a77
4 changed files with 42 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
backend/onedrive/api/types.go
View File

@ -100,8 +100,9 @@ type FolderFacet struct {
// HashesType groups different types of hashes into a single structure, for an item on OneDrive.
type HashesType struct {
Sha1Hash string `json:"sha1Hash"` // base64 encoded SHA1 hash for the contents of the file (if available)
Crc32Hash string `json:"crc32Hash"` // base64 encoded CRC32 value of the file (if available)
Sha1Hash string `json:"sha1Hash"` // hex encoded SHA1 hash for the contents of the file (if available)
Crc32Hash string `json:"crc32Hash"` // hex encoded CRC32 value of the file (if available)
QuickXorHash string `json:"quickXorHash"` // base64 encoded QuickXorHash value of the file (if available)
}
// FileFacet groups file-related data on OneDrive into a single structure.

44
backend/onedrive/onedrive.go
View File

@ -3,6 +3,8 @@
package onedrive
import (
"encoding/base64"
"encoding/hex"
"encoding/json"
"fmt"
"io"
@ -245,14 +247,15 @@ type Fs struct {
//
// Will definitely have info but maybe not meta
type Object struct {
fs *Fs // what this object is part of
remote string // The remote path
hasMetaData bool // whether info below has been set
size int64 // size of the object
modTime time.Time // modification time of the object
id string // ID of the object
sha1 string // SHA-1 of the object content
mimeType string // Content-Type of object from server (may not be as uploaded)
fs *Fs // what this object is part of
remote string // The remote path
hasMetaData bool // whether info below has been set
size int64 // size of the object
modTime time.Time // modification time of the object
id string // ID of the object
sha1 string // SHA-1 of the object content
quickxorhash string // QuickXorHash of the object content
mimeType string // Content-Type of object from server (may not be as uploaded)
}
// ------------------------------------------------------------
@ -949,6 +952,9 @@ func (f *Fs) About() (usage *fs.Usage, err error) {
// Hashes returns the supported hash sets.
func (f *Fs) Hashes() hash.Set {
if f.isBusiness {
return hash.Set(hash.QuickXorHash)
}
return hash.Set(hash.SHA1)
}
@ -979,6 +985,12 @@ func (o *Object) srvPath() string {
// Hash returns the SHA-1 of an object returning a lowercase hex string
func (o *Object) Hash(t hash.Type) (string, error) {
if o.fs.isBusiness {
if t != hash.QuickXorHash {
return "", hash.ErrUnsupported
}
return o.quickxorhash, nil
}
if t != hash.SHA1 {
return "", hash.ErrUnsupported
}
@ -1003,18 +1015,22 @@ func (o *Object) setMetaData(info *api.Item) (err error) {
o.hasMetaData = true
o.size = info.Size
// Docs: https://dev.onedrive.com/facets/hashes_facet.htm
// Docs: https://docs.microsoft.com/en-us/onedrive/developer/rest-api/resources/hashes
//
// The docs state both that the hashes are returned as hex
// strings, and as base64 strings. Testing reveals they are in
// fact uppercase hex strings.
//
// In OneDrive for Business, SHA1 and CRC32 hash values are not returned for files.
// We use SHA1 for onedrive personal and QuickXorHash for onedrive for business
if info.File != nil {
o.mimeType = info.File.MimeType
if info.File.Hashes.Sha1Hash != "" {
o.sha1 = strings.ToLower(info.File.Hashes.Sha1Hash)
}
if info.File.Hashes.QuickXorHash != "" {
h, err := base64.StdEncoding.DecodeString(info.File.Hashes.QuickXorHash)
if err != nil {
fs.Errorf(o, "Failed to decode QuickXorHash %q: %v", info.File.Hashes.QuickXorHash, err)
} else {
o.quickxorhash = hex.EncodeToString(h)
}
}
}
if info.FileSystemInfo != nil {
o.modTime = time.Time(info.FileSystemInfo.LastModifiedDateTime)

5
docs/content/onedrive.md
View File

@ -133,8 +133,11 @@ OneDrive allows modification times to be set on objects accurate to 1
second. These will be used to detect whether objects need syncing or
not.
One drive supports SHA1 type hashes, so you can use `--checksum` flag.
OneDrive personal supports SHA1 type hashes. OneDrive for business and
Sharepoint Server support
[QuickXorHash](https://docs.microsoft.com/en-us/onedrive/developer/code-snippets/quickxorhash).
For all types of OneDrive you can use the `--checksum` flag.
### Deleting files ###

6
docs/content/overview.md
View File

@ -29,7 +29,7 @@ Here is an overview of the major features of each cloud storage system.
| Hubic | MD5 | Yes | No | No | R/W |
| Mega | - | No | No | Yes | - |
| Microsoft Azure Blob Storage | MD5 | Yes | No | No | R/W |
| Microsoft OneDrive | SHA1 | Yes | Yes | No | R |
| Microsoft OneDrive | SHA1 ‡‡ | Yes | Yes | No | R |
| Openstack Swift | MD5 | Yes | No | No | R/W |
| pCloud | MD5, SHA1 | Yes | No | No | W |
| QingStor | MD5 | No | No | No | R/W |
@ -57,6 +57,10 @@ or `sha1sum` as well as `echo` are in the remote's PATH.
†† WebDAV supports modtimes when used with Owncloud and Nextcloud only.
‡‡ Microsoft OneDrive Personal supports SHA1 hashes, whereas OneDrive
for business and SharePoint server support Microsoft's own
[QuickXorHash](https://docs.microsoft.com/en-us/onedrive/developer/code-snippets/quickxorhash).
### ModTime ###
The cloud storage system supports setting modification times on