community
/
rclone
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

webdav: support MD5 and SHA1 hashes with Owncloud and Nextcloud - fixes #2379

This commit is contained in:
Nick Craig-Wood 2019-01-27 13:33:21 +00:00
parent 53a8b5a275
commit 8a774a3dd4
4 changed files with 96 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
backend/webdav/api/types.go
View File

@ -10,6 +10,7 @@ import (
"time"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fs/hash"
)
const (
@ -65,11 +66,12 @@ type Response struct {
// Note that status collects all the status values for which we just
// check the first is OK.
type Prop struct {
Status []string `xml:"DAV: status"`
Name string `xml:"DAV: prop>displayname,omitempty"`
Type *xml.Name `xml:"DAV: prop>resourcetype>collection,omitempty"`
Size int64 `xml:"DAV: prop>getcontentlength,omitempty"`
Modified Time `xml:"DAV: prop>getlastmodified,omitempty"`
Status []string `xml:"DAV: status"`
Name string `xml:"DAV: prop>displayname,omitempty"`
Type *xml.Name `xml:"DAV: prop>resourcetype>collection,omitempty"`
Size int64 `xml:"DAV: prop>getcontentlength,omitempty"`
Modified Time `xml:"DAV: prop>getlastmodified,omitempty"`
Checksums []string `xml:"prop>checksums>checksum,omitempty"`
}
// Parse a status of the form "HTTP/1.1 200 OK" or "HTTP/1.1 200"
@ -95,6 +97,26 @@ func (p *Prop) StatusOK() bool {
return false
}
// Hashes returns a map of all checksums - may be nil
func (p *Prop) Hashes() (hashes map[hash.Type]string) {
if len(p.Checksums) == 0 {
return nil
}
hashes = make(map[hash.Type]string)
for _, checksums := range p.Checksums {
checksums = strings.ToLower(checksums)
for _, checksum := range strings.Split(checksums, " ") {
switch {
case strings.HasPrefix(checksum, "sha1:"):
hashes[hash.SHA1] = checksum[5:]
case strings.HasPrefix(checksum, "md5:"):
hashes[hash.MD5] = checksum[4:]
}
}
}
return hashes
}
// PropValue is a tagged name and value
type PropValue struct {
XMLName xml.Name `xml:""`

79
backend/webdav/webdav.go
View File

@ -2,23 +2,13 @@
// object storage system.
package webdav
// Owncloud: Getting Oc-Checksum:
// SHA1:f572d396fae9206628714fb2ce00f72e94f2258f on HEAD but not on
// nextcloud?
// docs for file webdav
// https://docs.nextcloud.com/server/12/developer_manual/client_apis/WebDAV/index.html
// indicates checksums can be set as metadata here
// https://github.com/nextcloud/server/issues/6129
// owncloud seems to have checksums as metadata though - can read them
// SetModTime might be possible
// https://stackoverflow.com/questions/3579608/webdav-can-a-client-modify-the-mtime-of-a-file
// ...support for a PROPSET to lastmodified (mind the missing get) which does the utime() call might be an option.
// For example the ownCloud WebDAV server does it that way.
import (
"bytes"
"encoding/xml"
"fmt"
"io"
@ -116,6 +106,7 @@ type Fs struct {
canStream bool // set if can stream
useOCMtime bool // set if can use X-OC-Mtime
retryWithZeroDepth bool // some vendors (sharepoint) won't list files when Depth is 1 (our default)
hasChecksums bool // set if can use owncloud style checksums
}
// Object describes a webdav object
@ -127,7 +118,8 @@ type Object struct {
hasMetaData bool // whether info below has been set
size int64 // size of the object
modTime time.Time // modification time of the object
sha1 string // SHA-1 of the object content
sha1 string // SHA-1 of the object content if known
md5 string // MD5 of the object content if known
}
// ------------------------------------------------------------
@ -194,6 +186,9 @@ func (f *Fs) readMetaDataForPath(path string, depth string) (info *api.Prop, err
},
NoRedirect: true,
}
if f.hasChecksums {
opts.Body = bytes.NewBuffer(owncloudProps)
}
var result api.Multistatus
var resp *http.Response
err = f.pacer.Call(func() (bool, error) {
@ -357,9 +352,11 @@ func (f *Fs) setQuirks(vendor string) error {
f.canStream = true
f.precision = time.Second
f.useOCMtime = true
f.hasChecksums = true
case "nextcloud":
f.precision = time.Second
f.useOCMtime = true
f.hasChecksums = true
case "sharepoint":
// To mount sharepoint, two Cookies are required
// They have to be set instead of BasicAuth
@ -426,6 +423,22 @@ func (f *Fs) NewObject(remote string) (fs.Object, error) {
return f.newObjectWithInfo(remote, nil)
}
// Read the normal props, plus the checksums
//
// <oc:checksums><oc:checksum>SHA1:f572d396fae9206628714fb2ce00f72e94f2258f MD5:b1946ac92492d2347c6235b4d2611184 ADLER32:084b021f</oc:checksum></oc:checksums>
var owncloudProps = []byte(`<?xml version="1.0"?>
<d:propfind xmlns:d="DAV:" xmlns:oc="http://owncloud.org/ns" xmlns:nc="http://nextcloud.org/ns">
<d:prop>
<d:displayname />
<d:getlastmodified />
<d:getcontentlength />
<d:resourcetype />
<d:getcontenttype />
<oc:checksums />
</d:prop>
</d:propfind>
`)
// list the objects into the function supplied
//
// If directories is set it only sends directories
@ -445,6 +458,9 @@ func (f *Fs) listAll(dir string, directoriesOnly bool, filesOnly bool, depth str
"Depth": depth,
},
}
if f.hasChecksums {
opts.Body = bytes.NewBuffer(owncloudProps)
}
var result api.Multistatus
var resp *http.Response
err = f.pacer.Call(func() (bool, error) {
@ -847,6 +863,9 @@ func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
// Hashes returns the supported hash sets.
func (f *Fs) Hashes() hash.Set {
if f.hasChecksums {
return hash.NewHashSet(hash.MD5, hash.SHA1)
}
return hash.Set(hash.None)
}
@ -870,12 +889,17 @@ func (o *Object) Remote() string {
return o.remote
}
// Hash returns the SHA-1 of an object returning a lowercase hex string
// Hash returns the SHA1 or MD5 of an object returning a lowercase hex string
func (o *Object) Hash(t hash.Type) (string, error) {
if t != hash.SHA1 {
return "", hash.ErrUnsupported
if o.fs.hasChecksums {
switch t {
case hash.SHA1:
return o.sha1, nil
case hash.MD5:
return o.md5, nil
}
}
return o.sha1, nil
return "", hash.ErrUnsupported
}
// Size returns the size of an object in bytes
@ -893,6 +917,11 @@ func (o *Object) setMetaData(info *api.Prop) (err error) {
o.hasMetaData = true
o.size = info.Size
o.modTime = time.Time(info.Modified)
if o.fs.hasChecksums {
hashes := info.Hashes()
o.sha1 = hashes[hash.SHA1]
o.md5 = hashes[hash.MD5]
}
return nil
}
@ -972,9 +1001,21 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
ContentLength: &size, // FIXME this isn't necessary with owncloud - See https://github.com/nextcloud/nextcloud-snap/issues/365
ContentType: fs.MimeType(src),
}
if o.fs.useOCMtime {
opts.ExtraHeaders = map[string]string{
"X-OC-Mtime": fmt.Sprintf("%f", float64(src.ModTime().UnixNano())/1E9),
if o.fs.useOCMtime || o.fs.hasChecksums {
opts.ExtraHeaders = map[string]string{}
if o.fs.useOCMtime {
opts.ExtraHeaders["X-OC-Mtime"] = fmt.Sprintf("%f", float64(src.ModTime().UnixNano())/1E9)
}
if o.fs.hasChecksums {
// Set an upload checksum - prefer SHA1
//
// This is used as an upload integrity test. If we set
// only SHA1 here, owncloud will calculate the MD5 too.
if sha1, _ := src.Hash(hash.SHA1); sha1 != "" {
opts.ExtraHeaders["OC-Checksum"] = "SHA1:" + sha1
} else if md5, _ := src.Hash(hash.MD5); md5 != "" {
opts.ExtraHeaders["OC-Checksum"] = "MD5:" + md5
}
}
}
err = o.fs.pacer.CallNoRetry(func() (bool, error) {

6
docs/content/overview.md
View File

@ -36,7 +36,7 @@ Here is an overview of the major features of each cloud storage system.
| pCloud | MD5, SHA1 | Yes | No | No | W |
| QingStor | MD5 | No | No | No | R/W |
| SFTP | MD5, SHA1 ‡ | Yes | Depends | No | - |
| WebDAV | - | Yes †† | Depends | No | - |
| WebDAV | MD5, SHA1 ††| Yes ††† | Depends | No | - |
| Yandex Disk | MD5 | Yes | No | No | R/W |
| The local filesystem | All | Yes | Depends | No | - |
@ -57,7 +57,9 @@ This is an SHA256 sum of all the 4MB block SHA256s.
‡ SFTP supports checksums if the same login has shell access and `md5sum`
or `sha1sum` as well as `echo` are in the remote's PATH.
†† WebDAV supports modtimes when used with Owncloud and Nextcloud only.
†† WebDAV supports hashes when used with Owncloud and Nextcloud only.
††† WebDAV supports modtimes when used with Owncloud and Nextcloud only.
‡‡ Microsoft OneDrive Personal supports SHA1 hashes, whereas OneDrive
for business and SharePoint server support Microsoft's own

6
docs/content/webdav.md
View File

@ -99,7 +99,11 @@ To copy a local directory to an WebDAV directory called backup
Plain WebDAV does not support modified times. However when used with
Owncloud or Nextcloud rclone will support modified times.
Hashes are not supported.
Likewise plain WebDAV does not support hashes, however when used with
Owncloud or Nexcloud rclone will support SHA1 and MD5 hashes.
Depending on the exact version of Owncloud or Nextcloud hashes may
appear on all objects, or only on objects which had a hash uploaded
with them.
<!--- autogenerated options start - DO NOT EDIT, instead edit fs.RegInfo in backend/webdav/webdav.go then run make backenddocs -->
### Standard Options