webdav: add "fastmail" provider for Fastmail Files

This provider:

- supports the `X-OC-Mtime` header to set the mtime

- calculates SHA1 checksum server side and returns it as a `ME:sha1hex` prop

To differentiate the new hasMESHA1 quirk, the existing hasMD5 and hasSHA1
quirks for Owncloud have been renamed to hasOCMD5 and hasOCSHA1.

Fixes #6837
This commit is contained in:
Arnavion 2023-03-12 21:45:54 -07:00 committed by Nick Craig-Wood
parent 0e134364ac
commit 29fe0177bd
6 changed files with 74 additions and 45 deletions

View File

@ -37,6 +37,7 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
* Dreamhost [:page_facing_up:](https://rclone.org/s3/#dreamhost) * Dreamhost [:page_facing_up:](https://rclone.org/s3/#dreamhost)
* Dropbox [:page_facing_up:](https://rclone.org/dropbox/) * Dropbox [:page_facing_up:](https://rclone.org/dropbox/)
* Enterprise File Fabric [:page_facing_up:](https://rclone.org/filefabric/) * Enterprise File Fabric [:page_facing_up:](https://rclone.org/filefabric/)
* Fastmail Files [:page_facing_up:](https://rclone.org/webdav/#fastmail-files)
* FTP [:page_facing_up:](https://rclone.org/ftp/) * FTP [:page_facing_up:](https://rclone.org/ftp/)
* Google Cloud Storage [:page_facing_up:](https://rclone.org/googlecloudstorage/) * Google Cloud Storage [:page_facing_up:](https://rclone.org/googlecloudstorage/)
* Google Drive [:page_facing_up:](https://rclone.org/drive/) * Google Drive [:page_facing_up:](https://rclone.org/drive/)

View File

@ -75,6 +75,7 @@ type Prop struct {
Size int64 `xml:"DAV: prop>getcontentlength,omitempty"` Size int64 `xml:"DAV: prop>getcontentlength,omitempty"`
Modified Time `xml:"DAV: prop>getlastmodified,omitempty"` Modified Time `xml:"DAV: prop>getlastmodified,omitempty"`
Checksums []string `xml:"prop>checksums>checksum,omitempty"` Checksums []string `xml:"prop>checksums>checksum,omitempty"`
MESha1Hex *string `xml:"ME: prop>sha1hex,omitempty"` // Fastmail-specific sha1 checksum
} }
// Parse a status of the form "HTTP/1.1 200 OK" or "HTTP/1.1 200" // Parse a status of the form "HTTP/1.1 200 OK" or "HTTP/1.1 200"
@ -102,9 +103,7 @@ func (p *Prop) StatusOK() bool {
// Hashes returns a map of all checksums - may be nil // Hashes returns a map of all checksums - may be nil
func (p *Prop) Hashes() (hashes map[hash.Type]string) { func (p *Prop) Hashes() (hashes map[hash.Type]string) {
if len(p.Checksums) == 0 { if len(p.Checksums) > 0 {
return nil
}
hashes = make(map[hash.Type]string) hashes = make(map[hash.Type]string)
for _, checksums := range p.Checksums { for _, checksums := range p.Checksums {
checksums = strings.ToLower(checksums) checksums = strings.ToLower(checksums)
@ -118,6 +117,13 @@ func (p *Prop) Hashes() (hashes map[hash.Type]string) {
} }
} }
return hashes return hashes
} else if p.MESha1Hex != nil {
hashes = make(map[hash.Type]string)
hashes[hash.SHA1] = *p.MESha1Hex
return hashes
} else {
return nil
}
} }
// PropValue is a tagged name and value // PropValue is a tagged name and value

View File

@ -76,6 +76,9 @@ func init() {
Name: "vendor", Name: "vendor",
Help: "Name of the WebDAV site/service/software you are using.", Help: "Name of the WebDAV site/service/software you are using.",
Examples: []fs.OptionExample{{ Examples: []fs.OptionExample{{
Value: "fastmail",
Help: "Fastmail Files",
}, {
Value: "nextcloud", Value: "nextcloud",
Help: "Nextcloud", Help: "Nextcloud",
}, { }, {
@ -155,8 +158,9 @@ type Fs struct {
useOCMtime bool // set if can use X-OC-Mtime useOCMtime bool // set if can use X-OC-Mtime
retryWithZeroDepth bool // some vendors (sharepoint) won't list files when Depth is 1 (our default) retryWithZeroDepth bool // some vendors (sharepoint) won't list files when Depth is 1 (our default)
checkBeforePurge bool // enables extra check that directory to purge really exists checkBeforePurge bool // enables extra check that directory to purge really exists
hasMD5 bool // set if can use owncloud style checksums for MD5 hasOCMD5 bool // set if can use owncloud style checksums for MD5
hasSHA1 bool // set if can use owncloud style checksums for SHA1 hasOCSHA1 bool // set if can use owncloud style checksums for SHA1
hasMESHA1 bool // set if can use fastmail style checksums for SHA1
ntlmAuthMu sync.Mutex // mutex to serialize NTLM auth roundtrips ntlmAuthMu sync.Mutex // mutex to serialize NTLM auth roundtrips
} }
@ -278,7 +282,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string, depth string)
}, },
NoRedirect: true, NoRedirect: true,
} }
if f.hasMD5 || f.hasSHA1 { if f.hasOCMD5 || f.hasOCSHA1 {
opts.Body = bytes.NewBuffer(owncloudProps) opts.Body = bytes.NewBuffer(owncloudProps)
} }
var result api.Multistatus var result api.Multistatus
@ -546,16 +550,21 @@ func (f *Fs) fetchAndSetBearerToken() error {
// setQuirks adjusts the Fs for the vendor passed in // setQuirks adjusts the Fs for the vendor passed in
func (f *Fs) setQuirks(ctx context.Context, vendor string) error { func (f *Fs) setQuirks(ctx context.Context, vendor string) error {
switch vendor { switch vendor {
case "fastmail":
f.canStream = true
f.precision = time.Second
f.useOCMtime = true
f.hasMESHA1 = true
case "owncloud": case "owncloud":
f.canStream = true f.canStream = true
f.precision = time.Second f.precision = time.Second
f.useOCMtime = true f.useOCMtime = true
f.hasMD5 = true f.hasOCMD5 = true
f.hasSHA1 = true f.hasOCSHA1 = true
case "nextcloud": case "nextcloud":
f.precision = time.Second f.precision = time.Second
f.useOCMtime = true f.useOCMtime = true
f.hasSHA1 = true f.hasOCSHA1 = true
case "sharepoint": case "sharepoint":
// To mount sharepoint, two Cookies are required // To mount sharepoint, two Cookies are required
// They have to be set instead of BasicAuth // They have to be set instead of BasicAuth
@ -667,7 +676,7 @@ func (f *Fs) listAll(ctx context.Context, dir string, directoriesOnly bool, file
"Depth": depth, "Depth": depth,
}, },
} }
if f.hasMD5 || f.hasSHA1 { if f.hasOCMD5 || f.hasOCSHA1 {
opts.Body = bytes.NewBuffer(owncloudProps) opts.Body = bytes.NewBuffer(owncloudProps)
} }
var result api.Multistatus var result api.Multistatus
@ -1126,10 +1135,10 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
// Hashes returns the supported hash sets. // Hashes returns the supported hash sets.
func (f *Fs) Hashes() hash.Set { func (f *Fs) Hashes() hash.Set {
hashes := hash.Set(hash.None) hashes := hash.Set(hash.None)
if f.hasMD5 { if f.hasOCMD5 {
hashes.Add(hash.MD5) hashes.Add(hash.MD5)
} }
if f.hasSHA1 { if f.hasOCSHA1 || f.hasMESHA1 {
hashes.Add(hash.SHA1) hashes.Add(hash.SHA1)
} }
return hashes return hashes
@ -1197,10 +1206,10 @@ func (o *Object) Remote() string {
// Hash returns the SHA1 or MD5 of an object returning a lowercase hex string // Hash returns the SHA1 or MD5 of an object returning a lowercase hex string
func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) { func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
if t == hash.MD5 && o.fs.hasMD5 { if t == hash.MD5 && o.fs.hasOCMD5 {
return o.md5, nil return o.md5, nil
} }
if t == hash.SHA1 && o.fs.hasSHA1 { if t == hash.SHA1 && (o.fs.hasOCSHA1 || o.fs.hasMESHA1) {
return o.sha1, nil return o.sha1, nil
} }
return "", hash.ErrUnsupported return "", hash.ErrUnsupported
@ -1222,12 +1231,12 @@ func (o *Object) setMetaData(info *api.Prop) (err error) {
o.hasMetaData = true o.hasMetaData = true
o.size = info.Size o.size = info.Size
o.modTime = time.Time(info.Modified) o.modTime = time.Time(info.Modified)
if o.fs.hasMD5 || o.fs.hasSHA1 { if o.fs.hasOCMD5 || o.fs.hasOCSHA1 || o.fs.hasMESHA1 {
hashes := info.Hashes() hashes := info.Hashes()
if o.fs.hasSHA1 { if o.fs.hasOCSHA1 || o.fs.hasMESHA1 {
o.sha1 = hashes[hash.SHA1] o.sha1 = hashes[hash.SHA1]
} }
if o.fs.hasMD5 { if o.fs.hasOCMD5 {
o.md5 = hashes[hash.MD5] o.md5 = hashes[hash.MD5]
} }
} }
@ -1315,7 +1324,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
ContentType: fs.MimeType(ctx, src), ContentType: fs.MimeType(ctx, src),
Options: options, Options: options,
} }
if o.fs.useOCMtime || o.fs.hasMD5 || o.fs.hasSHA1 { if o.fs.useOCMtime || o.fs.hasOCMD5 || o.fs.hasOCSHA1 {
opts.ExtraHeaders = map[string]string{} opts.ExtraHeaders = map[string]string{}
if o.fs.useOCMtime { if o.fs.useOCMtime {
opts.ExtraHeaders["X-OC-Mtime"] = fmt.Sprintf("%d", src.ModTime(ctx).Unix()) opts.ExtraHeaders["X-OC-Mtime"] = fmt.Sprintf("%d", src.ModTime(ctx).Unix())
@ -1323,12 +1332,12 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
// Set one upload checksum // Set one upload checksum
// Owncloud uses one checksum only to check the upload and stores its own SHA1 and MD5 // Owncloud uses one checksum only to check the upload and stores its own SHA1 and MD5
// Nextcloud stores the checksum you supply (SHA1 or MD5) but only stores one // Nextcloud stores the checksum you supply (SHA1 or MD5) but only stores one
if o.fs.hasSHA1 { if o.fs.hasOCSHA1 {
if sha1, _ := src.Hash(ctx, hash.SHA1); sha1 != "" { if sha1, _ := src.Hash(ctx, hash.SHA1); sha1 != "" {
opts.ExtraHeaders["OC-Checksum"] = "SHA1:" + sha1 opts.ExtraHeaders["OC-Checksum"] = "SHA1:" + sha1
} }
} }
if o.fs.hasMD5 && opts.ExtraHeaders["OC-Checksum"] == "" { if o.fs.hasOCMD5 && opts.ExtraHeaders["OC-Checksum"] == "" {
if md5, _ := src.Hash(ctx, hash.MD5); md5 != "" { if md5, _ := src.Hash(ctx, hash.MD5); md5 != "" {
opts.ExtraHeaders["OC-Checksum"] = "MD5:" + md5 opts.ExtraHeaders["OC-Checksum"] = "MD5:" + md5
} }

View File

@ -121,6 +121,7 @@ WebDAV or S3, that work out of the box.)
{{< provider name="Dreamhost" home="https://www.dreamhost.com/cloud/storage/" config="/s3/#dreamhost" >}} {{< provider name="Dreamhost" home="https://www.dreamhost.com/cloud/storage/" config="/s3/#dreamhost" >}}
{{< provider name="Dropbox" home="https://www.dropbox.com/" config="/dropbox/" >}} {{< provider name="Dropbox" home="https://www.dropbox.com/" config="/dropbox/" >}}
{{< provider name="Enterprise File Fabric" home="https://storagemadeeasy.com/about/" config="/filefabric/" >}} {{< provider name="Enterprise File Fabric" home="https://storagemadeeasy.com/about/" config="/filefabric/" >}}
{{< provider name="Fastmail Files" home="https://www.fastmail.com/" config="/webdav/#fastmail-files" >}}
{{< provider name="FTP" home="https://en.wikipedia.org/wiki/File_Transfer_Protocol" config="/ftp/" >}} {{< provider name="FTP" home="https://en.wikipedia.org/wiki/File_Transfer_Protocol" config="/ftp/" >}}
{{< provider name="Google Cloud Storage" home="https://cloud.google.com/storage/" config="/googlecloudstorage/" >}} {{< provider name="Google Cloud Storage" home="https://cloud.google.com/storage/" config="/googlecloudstorage/" >}}
{{< provider name="Google Drive" home="https://www.google.com/drive/" config="/drive/" >}} {{< provider name="Google Drive" home="https://www.google.com/drive/" config="/drive/" >}}

View File

@ -68,9 +68,9 @@ This is an SHA256 sum of all the 4 MiB block SHA256s.
² SFTP supports checksums if the same login has shell access and ² SFTP supports checksums if the same login has shell access and
`md5sum` or `sha1sum` as well as `echo` are in the remote's PATH. `md5sum` or `sha1sum` as well as `echo` are in the remote's PATH.
³ WebDAV supports hashes when used with Owncloud and Nextcloud only. ³ WebDAV supports hashes when used with Fastmail Files. Owncloud and Nextcloud only.
⁴ WebDAV supports modtimes when used with Owncloud and Nextcloud only. ⁴ WebDAV supports modtimes when used with Fastmail Files, Owncloud and Nextcloud only.
⁵ [QuickXorHash](https://docs.microsoft.com/en-us/onedrive/developer/code-snippets/quickxorhash) is Microsoft's own hash. ⁵ [QuickXorHash](https://docs.microsoft.com/en-us/onedrive/developer/code-snippets/quickxorhash) is Microsoft's own hash.

View File

@ -43,17 +43,19 @@ Choose a number from below, or type in your own value
url> https://example.com/remote.php/webdav/ url> https://example.com/remote.php/webdav/
Name of the WebDAV site/service/software you are using Name of the WebDAV site/service/software you are using
Choose a number from below, or type in your own value Choose a number from below, or type in your own value
1 / Nextcloud 1 / Fastmail Files
\ "nextcloud" \ (fastmail)
2 / Owncloud 2 / Nextcloud
\ "owncloud" \ (nextcloud)
3 / Sharepoint Online, authenticated by Microsoft account. 3 / Owncloud
\ "sharepoint" \ (owncloud)
4 / Sharepoint with NTLM authentication. Usually self-hosted or on-premises. 4 / Sharepoint Online, authenticated by Microsoft account
\ "sharepoint-ntlm" \ (sharepoint)
5 / Other site/service or software 5 / Sharepoint with NTLM authentication, usually self-hosted or on-premises
\ "other" \ (sharepoint-ntlm)
vendor> 1 6 / Other site/service or software
\ (other)
vendor> 2
User name User name
user> user user> user
Password. Password.
@ -100,10 +102,10 @@ To copy a local directory to an WebDAV directory called backup
### Modified time and hashes ### ### Modified time and hashes ###
Plain WebDAV does not support modified times. However when used with Plain WebDAV does not support modified times. However when used with
Owncloud or Nextcloud rclone will support modified times. Fastmail Files, Owncloud or Nextcloud rclone will support modified times.
Likewise plain WebDAV does not support hashes, however when used with Likewise plain WebDAV does not support hashes, however when used with
Owncloud or Nextcloud rclone will support SHA1 and MD5 hashes. Fastmail Files, Owncloud or Nextcloud rclone will support SHA1 and MD5 hashes.
Depending on the exact version of Owncloud or Nextcloud hashes may Depending on the exact version of Owncloud or Nextcloud hashes may
appear on all objects, or only on objects which had a hash uploaded appear on all objects, or only on objects which had a hash uploaded
with them. with them.
@ -242,6 +244,16 @@ Properties:
See below for notes on specific providers. See below for notes on specific providers.
## Fastmail Files
Use `https://webdav.fastmail.com/` or a subdirectory as the URL,
and your Fastmail email `username@domain.tld` as the username.
Follow [this documentation](https://www.fastmail.help/hc/en-us/articles/360058752854-App-passwords)
to create an app password with access to `Files (WebDAV)` and use
this as the password.
Fastmail supports modified times using the `X-OC-Mtime` header.
### Owncloud ### Owncloud
Click on the settings cog in the bottom right of the page and this Click on the settings cog in the bottom right of the page and this