azurefiles: finish docs and implementation and add optional interfaces
- use rclone's http Transport
- fix handling of 0 length files
- combine into one file and remove uneeded abstraction
- make `chunk_size` and `upload_concurrency` settable
- make auth the same as azureblob
- set the Features correctly
- implement `--azurefiles-max-stream-size`
- remove arbitrary sleep on Mkdir
- implement `--header-upload`
- implement read and write MimeType for objects
- implement optional methods
- About
- Copy
- DirMove
- Move
- OpenWriterAt
- PutStream
- finish documentation
- disable build on plan9 and js
Fixes #365
Fixes #7378
This commit is contained in:
Nick Craig-Wood
parent
b5301e03a6
commit
ddaf01ece9
|
|
@ -59,6 +59,7 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
|
|||
* Mega [:page_facing_up:](https://rclone.org/mega/)
|
||||
* Memory [:page_facing_up:](https://rclone.org/memory/)
|
||||
* Microsoft Azure Blob Storage [:page_facing_up:](https://rclone.org/azureblob/)
|
||||
* Microsoft Azure Files Storage [:page_facing_up:](https://rclone.org/azurefiles/)
|
||||
* Microsoft OneDrive [:page_facing_up:](https://rclone.org/onedrive/)
|
||||
* Minio [:page_facing_up:](https://rclone.org/s3/#minio)
|
||||
* Nextcloud [:page_facing_up:](https://rclone.org/webdav/#nextcloud)
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -1,3 +1,6 @@
|
|||
//go:build !plan9 && !js
|
||||
// +build !plan9,!js
|
||||
|
||||
package azurefiles
|
||||
|
||||
import (
|
||||
|
|
@ -41,7 +44,7 @@ func (f *Fs) InternalTestAuth(t *testing.T) {
|
|||
name: "SASUrl",
|
||||
options: &Options{
|
||||
ShareName: shareName,
|
||||
SASUrl: "",
|
||||
SASURL: "",
|
||||
}},
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,6 @@
|
|||
//go:build !plan9 && !js
|
||||
// +build !plan9,!js
|
||||
|
||||
package azurefiles
|
||||
|
||||
import (
|
||||
|
|
|
|||
|
|
@ -0,0 +1,7 @@
|
|||
// Build for azurefiles for unsupported platforms to stop go complaining
|
||||
// about "no buildable Go source files "
|
||||
|
||||
//go:build plan9 || js
|
||||
// +build plan9 js
|
||||
|
||||
package azurefiles
|
||||
|
|
@ -1,44 +0,0 @@
|
|||
package azurefiles
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Directory is a filesystem like directory provided by an Fs
|
||||
type Directory struct {
|
||||
common
|
||||
}
|
||||
|
||||
// Items returns the count of items in this directory or this
|
||||
// directory and subdirectories if known, -1 for unknown
|
||||
//
|
||||
// It is unknown since getting the count of items results in a
|
||||
// network request
|
||||
func (d *Directory) Items() int64 {
|
||||
return -1
|
||||
}
|
||||
|
||||
// ID returns empty string. Can be implemented as part of IDer
|
||||
func (d *Directory) ID() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// Size is returns the size of the file.
|
||||
// This method is implemented because it is part of the [fs.DirEntry] interface
|
||||
func (d *Directory) Size() int64 {
|
||||
return 0
|
||||
}
|
||||
|
||||
// ModTime returns the modification time of the object
|
||||
//
|
||||
// TODO: check whether FileLastWriteTime is what the clients of this API want. Maybe
|
||||
// FileLastWriteTime does not get changed when directory contents are updated but consumers
|
||||
// of this API expect d.ModTime to do so
|
||||
func (d *Directory) ModTime(ctx context.Context) time.Time {
|
||||
props, err := d.f.dirClient(d.remote).GetProperties(ctx, nil)
|
||||
if err != nil {
|
||||
return time.Now()
|
||||
}
|
||||
return *props.FileLastWriteTime
|
||||
}
|
||||
|
|
@ -1,292 +0,0 @@
|
|||
package azurefiles
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"path"
|
||||
"time"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azfile/directory"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azfile/file"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azfile/fileerror"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
)
|
||||
|
||||
const sleepDurationBetweenRecursiveMkdirPutCalls = time.Millisecond * 500
|
||||
const fourTbInBytes = 4398046511104
|
||||
|
||||
// NewObject finds the Object at remote. If it can't be found
|
||||
// it returns the error fs.ErrorObjectNotFound.
|
||||
//
|
||||
// Does not return ErrorIsDir when a directory exists instead of file. since the documentation
|
||||
// for [rclone.fs.Fs.NewObject] rqeuires no extra work to determine whether it is directory
|
||||
//
|
||||
// Inspired by azureblob store, this initiates a network request and returns an error if object is not found.
|
||||
func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
|
||||
fileClient := f.fileClient(remote)
|
||||
resp, err := fileClient.GetProperties(ctx, nil)
|
||||
if fileerror.HasCode(err, fileerror.ParentNotFound, fileerror.ResourceNotFound) {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
} else if err != nil {
|
||||
return nil, fmt.Errorf("unable to find object remote=%s : %w", remote, err)
|
||||
}
|
||||
|
||||
ob := objectInstance(f, remote, *resp.ContentLength, resp.ContentMD5, *resp.FileLastWriteTime)
|
||||
return &ob, nil
|
||||
}
|
||||
|
||||
// Mkdir creates nested directories as indicated by test FsMkdirRmdirSubdir
|
||||
// TODO: write custom test case where parent directories are created
|
||||
// Mkdir creates the container if it doesn't exist
|
||||
func (f *Fs) Mkdir(ctx context.Context, remote string) error {
|
||||
return f.mkdirRelativeToRootOfShare(ctx, f.decodedFullPath(remote))
|
||||
}
|
||||
|
||||
// rclone completes commands such as rclone copy localdir remote:parentcontainer/childcontainer
|
||||
// where localdir is a tree of files and directories. The above command is expected to complete even
|
||||
// when parentcontainer and childcontainer directors do not exist on the remote. The following
|
||||
// code with emphasis on fullPathRelativeToShareRoot is written to handle such cases by recursiely creating
|
||||
// parent directories all the way to the root of the share
|
||||
//
|
||||
// When path argument is an empty string, windows and linux return and error. However, this
|
||||
// implementation does not return an error
|
||||
func (f *Fs) mkdirRelativeToRootOfShare(ctx context.Context, fullPathRelativeToShareRoot string) error {
|
||||
fp := fullPathRelativeToShareRoot
|
||||
if fp == "" {
|
||||
return nil
|
||||
}
|
||||
dirClient := f.newSubdirectoryClientFromEncodedPathRelativeToShareRoot(f.encodePath(fp))
|
||||
// now := time.Now()
|
||||
// smbProps := &file.SMBProperties{
|
||||
// LastWriteTime: &now,
|
||||
// }
|
||||
// dirCreateOptions := &directory.CreateOptions{
|
||||
// FileSMBProperties: smbProps,
|
||||
// }
|
||||
|
||||
_, createDirErr := dirClient.Create(ctx, nil)
|
||||
if fileerror.HasCode(createDirErr, fileerror.ParentNotFound) {
|
||||
parentDir := path.Dir(fp)
|
||||
if parentDir == fp {
|
||||
log.Fatal("This will lead to infinite recursion since parent and remote are equal")
|
||||
}
|
||||
makeParentErr := f.mkdirRelativeToRootOfShare(ctx, parentDir)
|
||||
if makeParentErr != nil {
|
||||
return fmt.Errorf("could not make parent of %s : %w", fp, makeParentErr)
|
||||
}
|
||||
log.Printf("Mkdir: waiting for %s after making parent=%s", sleepDurationBetweenRecursiveMkdirPutCalls.String(), parentDir)
|
||||
time.Sleep(sleepDurationBetweenRecursiveMkdirPutCalls)
|
||||
return f.mkdirRelativeToRootOfShare(ctx, fp)
|
||||
} else if fileerror.HasCode(createDirErr, fileerror.ResourceAlreadyExists) {
|
||||
return nil
|
||||
} else if createDirErr != nil {
|
||||
return fmt.Errorf("unable to MkDir: %w", createDirErr)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Rmdir deletes the root folder
|
||||
//
|
||||
// Returns an error if it isn't empty
|
||||
func (f *Fs) Rmdir(ctx context.Context, remote string) error {
|
||||
dirClient := f.dirClient(remote)
|
||||
_, err := dirClient.Delete(ctx, nil)
|
||||
if err != nil {
|
||||
if fileerror.HasCode(err, fileerror.DirectoryNotEmpty) {
|
||||
return fs.ErrorDirectoryNotEmpty
|
||||
} else if fileerror.HasCode(err, fileerror.ResourceNotFound) {
|
||||
return fs.ErrorDirNotFound
|
||||
}
|
||||
return fmt.Errorf("could not rmdir dir=\"%s\" : %w", remote, err)
|
||||
}
|
||||
return nil
|
||||
|
||||
}
|
||||
|
||||
// Put the object
|
||||
//
|
||||
// Copies the reader in to the new object. This new object is returned.
|
||||
//
|
||||
// The new object may have been created if an error is returned
|
||||
// TODO: when file.CLient.Creat is being used, provide HTTP headesr such as content type and content MD5
|
||||
// TODO: maybe replace PUT with NewObject + Update
|
||||
// TODO: in case file is created but there is a problem on upload, what happens
|
||||
// TODO: what happens when file already exists at the location
|
||||
func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
||||
if src.Size() > fourTbInBytes {
|
||||
return nil, fmt.Errorf("max supported file size is 4TB. provided size is %d", src.Size())
|
||||
} else if src.Size() < 0 {
|
||||
// TODO: what should happened when src.Size == 0
|
||||
return nil, fmt.Errorf("src.Size is a required to be a whole number : %d", src.Size())
|
||||
}
|
||||
fc := f.fileClient(src.Remote())
|
||||
|
||||
_, createErr := fc.Create(ctx, src.Size(), nil)
|
||||
if fileerror.HasCode(createErr, fileerror.ParentNotFound) {
|
||||
parentDir := path.Dir(src.Remote())
|
||||
if mkDirErr := f.Mkdir(ctx, parentDir); mkDirErr != nil {
|
||||
return nil, fmt.Errorf("unable to make parent directories : %w", mkDirErr)
|
||||
}
|
||||
log.Printf("Mkdir: waiting for %s after making parent=%s", sleepDurationBetweenRecursiveMkdirPutCalls.String(), parentDir)
|
||||
time.Sleep(sleepDurationBetweenRecursiveMkdirPutCalls)
|
||||
return f.Put(ctx, in, src, options...)
|
||||
} else if createErr != nil {
|
||||
return nil, fmt.Errorf("unable to create file : %w", createErr)
|
||||
}
|
||||
|
||||
obj := &Object{
|
||||
common: common{
|
||||
f: f,
|
||||
remote: src.Remote(),
|
||||
},
|
||||
}
|
||||
if updateErr := obj.upload(ctx, in, src, true, options...); updateErr != nil {
|
||||
err := fmt.Errorf("while executing update after creating file as part of fs.Put : %w", updateErr)
|
||||
if _, delErr := fc.Delete(ctx, nil); delErr != nil {
|
||||
return nil, errors.Join(delErr, updateErr)
|
||||
}
|
||||
return obj, err
|
||||
}
|
||||
|
||||
return obj, nil
|
||||
}
|
||||
|
||||
// Name of the remote (as passed into NewFs)
|
||||
func (f *Fs) Name() string {
|
||||
return f.name
|
||||
}
|
||||
|
||||
// Root of the remote (as passed into NewFs)
|
||||
func (f *Fs) Root() string {
|
||||
return f.root
|
||||
}
|
||||
|
||||
// String converts this Fs to a string
|
||||
func (f *Fs) String() string {
|
||||
return fmt.Sprintf("azurefiles root '%s'", f.root)
|
||||
}
|
||||
|
||||
// Precision return the precision of this Fs
|
||||
//
|
||||
// One second. FileREST API times are in RFC1123 which in the example shows a precision of seconds
|
||||
// Source: https://learn.microsoft.com/en-us/rest/api/storageservices/representation-of-date-time-values-in-headers
|
||||
func (f *Fs) Precision() time.Duration {
|
||||
return time.Second
|
||||
}
|
||||
|
||||
// Hashes returns the supported hash sets.
|
||||
//
|
||||
// MD5: since it is listed as header in the response for file properties
|
||||
// Source: https://learn.microsoft.com/en-us/rest/api/storageservices/get-file-properties
|
||||
func (f *Fs) Hashes() hash.Set {
|
||||
return hash.NewHashSet(hash.MD5)
|
||||
}
|
||||
|
||||
// Features returns the optional features of this Fs
|
||||
//
|
||||
// TODO: add features:- public link, SlowModTime, SlowHash,
|
||||
// ReadMetadata, WriteMetadata,UserMetadata,PutUnchecked, PutStream
|
||||
// PartialUploads: Maybe????
|
||||
// FileID and DirectoryID can be implemented. They are atleast returned as part of listing response
|
||||
func (f *Fs) Features() *fs.Features {
|
||||
return &fs.Features{
|
||||
CanHaveEmptyDirectories: true,
|
||||
// Copy: func(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
|
||||
// return f.CopyFile(ctx, src, remote)
|
||||
// },
|
||||
}
|
||||
}
|
||||
|
||||
// List the objects and directories in dir into entries. The entries can be
|
||||
// returned in any order but should be for a complete directory.
|
||||
//
|
||||
// dir should be "" to list the root, and should not have trailing slashes.
|
||||
//
|
||||
// This should return ErrDirNotFound if the directory isn't found.
|
||||
//
|
||||
// TODO: handle case regariding "" and "/". I remember reading about them somewhere
|
||||
func (f *Fs) List(ctx context.Context, remote string) (fs.DirEntries, error) {
|
||||
var entries fs.DirEntries
|
||||
subDirClient := f.dirClient(remote)
|
||||
|
||||
// Checking whether directory exists
|
||||
_, err := subDirClient.GetProperties(ctx, nil)
|
||||
if fileerror.HasCode(err, fileerror.ParentNotFound, fileerror.ResourceNotFound) {
|
||||
return entries, fs.ErrorDirNotFound
|
||||
} else if err != nil {
|
||||
return entries, err
|
||||
}
|
||||
|
||||
pager := subDirClient.NewListFilesAndDirectoriesPager(listFilesAndDirectoriesOptions)
|
||||
for pager.More() {
|
||||
resp, err := pager.NextPage(ctx)
|
||||
if err != nil {
|
||||
return entries, err
|
||||
}
|
||||
for _, dir := range resp.Segment.Directories {
|
||||
de := &Directory{
|
||||
common{f: f,
|
||||
remote: path.Join(remote, f.decodePath(*dir.Name)),
|
||||
properties: properties{
|
||||
lastWriteTime: *dir.Properties.LastWriteTime,
|
||||
}},
|
||||
}
|
||||
entries = append(entries, de)
|
||||
}
|
||||
|
||||
for _, file := range resp.Segment.Files {
|
||||
de := &Object{
|
||||
common{f: f,
|
||||
remote: path.Join(remote, f.decodePath(*file.Name)),
|
||||
properties: properties{
|
||||
contentLength: *file.Properties.ContentLength,
|
||||
lastWriteTime: *file.Properties.LastWriteTime,
|
||||
}},
|
||||
}
|
||||
entries = append(entries, de)
|
||||
}
|
||||
}
|
||||
|
||||
return entries, nil
|
||||
|
||||
}
|
||||
|
||||
type encodedPath string
|
||||
|
||||
func (f *Fs) decodedFullPath(decodedRemote string) string {
|
||||
return path.Join(f.root, decodedRemote)
|
||||
}
|
||||
|
||||
func (f *Fs) dirClient(decodedRemote string) *directory.Client {
|
||||
fullPathDecoded := f.decodedFullPath(decodedRemote)
|
||||
fullPathEncoded := f.encodePath(fullPathDecoded)
|
||||
return f.newSubdirectoryClientFromEncodedPathRelativeToShareRoot(fullPathEncoded)
|
||||
}
|
||||
|
||||
func (f *Fs) newSubdirectoryClientFromEncodedPathRelativeToShareRoot(p encodedPath) *directory.Client {
|
||||
return f.shareRootDirClient.NewSubdirectoryClient(string(p))
|
||||
}
|
||||
|
||||
func (f *Fs) fileClient(decodedRemote string) *file.Client {
|
||||
fullPathDecoded := f.decodedFullPath(decodedRemote)
|
||||
fullPathEncoded := f.encodePath(fullPathDecoded)
|
||||
return f.fileClientFromEncodedPathRelativeToShareRoot(fullPathEncoded)
|
||||
}
|
||||
|
||||
func (f *Fs) fileClientFromEncodedPathRelativeToShareRoot(p encodedPath) *file.Client {
|
||||
return f.shareRootDirClient.NewFileClient(string(p))
|
||||
}
|
||||
|
||||
func (f *Fs) encodePath(p string) encodedPath {
|
||||
return encodedPath(f.opt.Enc.FromStandardPath(p))
|
||||
}
|
||||
|
||||
func (f *Fs) decodePath(p string) string {
|
||||
return f.opt.Enc.ToStandardPath(p)
|
||||
}
|
||||
|
||||
// on 20231019 at 1324 work to be continued at trying to fix FAIL: TestIntegration/FsMkdir/FsPutFiles/FromRoot
|
||||
|
|
@ -1,279 +0,0 @@
|
|||
package azurefiles
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azfile/file"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
)
|
||||
|
||||
// TODO: maybe use this in the result of list. or replace all instances where object instances are created
|
||||
func objectInstance(f *Fs, remote string, contentLength int64, md5Hash []byte, lwt time.Time) Object {
|
||||
return Object{common: common{
|
||||
f: f,
|
||||
remote: remote,
|
||||
properties: properties{
|
||||
contentLength: contentLength,
|
||||
md5Hash: md5Hash,
|
||||
lastWriteTime: lwt,
|
||||
},
|
||||
}}
|
||||
}
|
||||
|
||||
// Size of object in bytes
|
||||
func (o *Object) Size() int64 {
|
||||
return o.properties.contentLength
|
||||
}
|
||||
|
||||
// Fs returns the parent Fs
|
||||
func (o *Object) Fs() fs.Info {
|
||||
return o.f
|
||||
}
|
||||
|
||||
// Hash returns the MD5 of an object returning a lowercase hex string
|
||||
//
|
||||
// May make a network request becaue the [fs.List] method does not
|
||||
// return MD5 hashes for DirEntry
|
||||
func (o *Object) Hash(ctx context.Context, ty hash.Type) (string, error) {
|
||||
if ty != hash.MD5 {
|
||||
return "", hash.ErrUnsupported
|
||||
}
|
||||
if len(o.common.properties.md5Hash) == 0 {
|
||||
props, err := o.fileClient().GetProperties(ctx, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("unable to fetch properties to determine hash")
|
||||
}
|
||||
o.common.properties.md5Hash = props.ContentMD5
|
||||
}
|
||||
return hex.EncodeToString(o.common.properties.md5Hash), nil
|
||||
}
|
||||
|
||||
// Storable returns a boolean showing whether this object storable
|
||||
func (o *Object) Storable() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// Object describes a Azure File Share File not a Directory
|
||||
type Object struct {
|
||||
common
|
||||
}
|
||||
|
||||
// These fields have pointer types because it seems to
|
||||
// TODO: descide whether these could be pointer or not
|
||||
type properties struct {
|
||||
contentLength int64
|
||||
md5Hash []byte
|
||||
lastWriteTime time.Time
|
||||
}
|
||||
|
||||
func (o *Object) fileClient() *file.Client {
|
||||
decodedFullPath := o.f.decodedFullPath(o.remote)
|
||||
fullEncodedPath := o.f.encodePath(decodedFullPath)
|
||||
return o.f.fileClientFromEncodedPathRelativeToShareRoot(fullEncodedPath)
|
||||
}
|
||||
|
||||
// SetModTime sets the modification time
|
||||
func (o *Object) SetModTime(ctx context.Context, t time.Time) error {
|
||||
smbProps := file.SMBProperties{
|
||||
LastWriteTime: &t,
|
||||
}
|
||||
setHeadersOptions := file.SetHTTPHeadersOptions{
|
||||
SMBProperties: &smbProps,
|
||||
}
|
||||
_, err := o.fileClient().SetHTTPHeaders(ctx, &setHeadersOptions)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to set modTime : %w", err)
|
||||
}
|
||||
o.lastWriteTime = t
|
||||
return nil
|
||||
}
|
||||
|
||||
// ModTime returns the modification time of the object
|
||||
//
|
||||
// Returns time.Now() if not present
|
||||
// TODO: convert o.lastWriteTime to *time.Time so that one can know when it has
|
||||
// been explicitly set
|
||||
func (o *Object) ModTime(ctx context.Context) time.Time {
|
||||
if o.lastWriteTime.Unix() <= 1 {
|
||||
return time.Now()
|
||||
}
|
||||
return o.lastWriteTime
|
||||
}
|
||||
|
||||
// Remove an object
|
||||
func (o *Object) Remove(ctx context.Context) error {
|
||||
// TODO: should the options for delete not be nil. Depends on behaviour expected by consumers
|
||||
if _, err := o.fileClient().Delete(ctx, nil); err != nil {
|
||||
return fmt.Errorf("unable to delete remote=\"%s\" : %w", o.remote, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Open an object for read
|
||||
//
|
||||
// TODO: check for mandatory options and the other options
|
||||
func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
|
||||
downloadStreamOptions := file.DownloadStreamOptions{}
|
||||
for _, opt := range options {
|
||||
switch v := opt.(type) {
|
||||
case *fs.SeekOption:
|
||||
httpRange := file.HTTPRange{
|
||||
Offset: v.Offset,
|
||||
}
|
||||
downloadStreamOptions.Range = httpRange
|
||||
case *fs.RangeOption:
|
||||
var start *int64
|
||||
var end *int64
|
||||
if v.Start >= 0 {
|
||||
start = &v.Start
|
||||
}
|
||||
if v.End >= 0 {
|
||||
end = &v.End
|
||||
}
|
||||
|
||||
fhr := file.HTTPRange{}
|
||||
if start != nil && end != nil {
|
||||
fhr.Offset = *start
|
||||
fhr.Count = *end - *start + 1
|
||||
} else if start != nil && end == nil {
|
||||
fhr.Offset = *start
|
||||
} else if start == nil && end != nil {
|
||||
fhr.Offset = o.contentLength - *end
|
||||
}
|
||||
|
||||
downloadStreamOptions.Range = fhr
|
||||
}
|
||||
}
|
||||
resp, err := o.fileClient().DownloadStream(ctx, &downloadStreamOptions)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not open remote=\"%s\" : %w", o.remote, err)
|
||||
}
|
||||
return resp.Body, nil
|
||||
}
|
||||
|
||||
func (o *Object) upload(ctx context.Context, in io.Reader, src fs.ObjectInfo, isDestNewlyCreated bool, options ...fs.OpenOption) error {
|
||||
if src.Size() > fourTbInBytes {
|
||||
return fmt.Errorf("max supported file size is 4TB. provided size is %d", src.Size())
|
||||
} else if src.Size() < 0 {
|
||||
return fmt.Errorf("files with unknown sizes are not supported")
|
||||
}
|
||||
|
||||
fc := o.fileClient()
|
||||
|
||||
if !isDestNewlyCreated {
|
||||
if src.Size() != o.Size() {
|
||||
if _, resizeErr := fc.Resize(ctx, src.Size(), nil); resizeErr != nil {
|
||||
return fmt.Errorf("unable to resize while trying to update. %w ", resizeErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var md5Hash []byte
|
||||
hashToBeComputed := false
|
||||
if hashStr, err := src.Hash(ctx, hash.MD5); err != nil || hashStr == "" {
|
||||
hashToBeComputed = true
|
||||
} else {
|
||||
var decodeErr error
|
||||
md5Hash, decodeErr = hex.DecodeString(hashStr)
|
||||
if decodeErr != nil {
|
||||
hashToBeComputed = true
|
||||
msg := fmt.Sprintf("should not happen. Error while decoding hex encoded md5 '%s'. Error is %s",
|
||||
hashStr, decodeErr.Error())
|
||||
slog.Error(msg)
|
||||
}
|
||||
}
|
||||
var uploadErr error
|
||||
if hashToBeComputed {
|
||||
md5Hash, uploadErr = uploadStreamAndComputeHash(ctx, fc, in, src, options...)
|
||||
} else {
|
||||
uploadErr = uploadStream(ctx, fc, in, src, options...)
|
||||
}
|
||||
if uploadErr != nil {
|
||||
return fmt.Errorf("while uploading %s : %w", src.Remote(), uploadErr)
|
||||
}
|
||||
|
||||
modTime := src.ModTime(ctx)
|
||||
if err := uploadSizeHashLWT(ctx, fc, src.Size(), md5Hash, modTime); err != nil {
|
||||
|
||||
return fmt.Errorf("while setting size hash and last write time for %s : %w", src.Remote(), err)
|
||||
}
|
||||
o.properties.contentLength = src.Size()
|
||||
o.properties.md5Hash = md5Hash
|
||||
o.properties.lastWriteTime = modTime
|
||||
return nil
|
||||
}
|
||||
|
||||
// Update the object with the contents of the io.Reader, modTime, size and MD5 hash
|
||||
// Does not create a new object
|
||||
//
|
||||
// TODO: implement options. understand purpose of options
|
||||
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
|
||||
return o.upload(ctx, in, src, false, options...)
|
||||
}
|
||||
|
||||
// cannot set modTime header here because setHTTPHeaders does not allow setting metadata
|
||||
func uploadStream(ctx context.Context, fc *file.Client, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
|
||||
// TODO: set concurrency level
|
||||
uploadStreamOptions := file.UploadStreamOptions{
|
||||
ChunkSize: chunkSize(options...),
|
||||
}
|
||||
|
||||
if err := fc.UploadStream(ctx, in, &uploadStreamOptions); err != nil {
|
||||
return fmt.Errorf("unable to upload. cannot upload stream : %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func uploadStreamAndComputeHash(ctx context.Context, fc *file.Client, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) ([]byte, error) {
|
||||
hasher := md5.New()
|
||||
teeReader := io.TeeReader(in, hasher)
|
||||
err := uploadStream(ctx, fc, teeReader, src, options...)
|
||||
if err != nil {
|
||||
return []byte{}, err
|
||||
}
|
||||
return hasher.Sum(nil), nil
|
||||
|
||||
}
|
||||
|
||||
// the function is named with prefix 'upload' since it indicates that things will be modified on the server
|
||||
func uploadSizeHashLWT(ctx context.Context, fc *file.Client, size int64, hash []byte, lwt time.Time) error {
|
||||
smbProps := file.SMBProperties{
|
||||
LastWriteTime: &lwt,
|
||||
}
|
||||
httpHeaders := &file.HTTPHeaders{
|
||||
ContentMD5: hash,
|
||||
}
|
||||
_, err := fc.SetHTTPHeaders(ctx, &file.SetHTTPHeadersOptions{
|
||||
FileContentLength: &size,
|
||||
SMBProperties: &smbProps,
|
||||
HTTPHeaders: httpHeaders,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("while setting size, hash, lastWriteTime : %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func chunkSize(options ...fs.OpenOption) int64 {
|
||||
for _, option := range options {
|
||||
if chunkOpt, ok := option.(*fs.ChunkOption); ok {
|
||||
return chunkOpt.ChunkSize
|
||||
}
|
||||
}
|
||||
return 1048576
|
||||
}
|
||||
|
||||
// Return a string version
|
||||
func (o *Object) String() string {
|
||||
if o == nil {
|
||||
return "<nil>"
|
||||
}
|
||||
return o.common.String()
|
||||
}
|
||||
|
|
@ -58,6 +58,7 @@ docs = [
|
|||
"memory.md",
|
||||
"netstorage.md",
|
||||
"azureblob.md",
|
||||
"azurefiles.md",
|
||||
"onedrive.md",
|
||||
"opendrive.md",
|
||||
"oracleobjectstorage.md",
|
||||
|
|
|
|||
|
|
@ -144,6 +144,7 @@ WebDAV or S3, that work out of the box.)
|
|||
{{< provider name="Mega" home="https://mega.nz/" config="/mega/" >}}
|
||||
{{< provider name="Memory" home="/memory/" config="/memory/" >}}
|
||||
{{< provider name="Microsoft Azure Blob Storage" home="https://azure.microsoft.com/en-us/services/storage/blobs/" config="/azureblob/" >}}
|
||||
{{< provider name="Microsoft Azure Files Storage" home="https://azure.microsoft.com/en-us/services/storage/files/" config="/azurefiles/" >}}
|
||||
{{< provider name="Microsoft OneDrive" home="https://onedrive.live.com/" config="/onedrive/" >}}
|
||||
{{< provider name="Minio" home="https://www.minio.io/" config="/s3/#minio" >}}
|
||||
{{< provider name="Nextcloud" home="https://nextcloud.com/" config="/webdav/#nextcloud" >}}
|
||||
|
|
|
|||
|
|
@ -1,21 +1,707 @@
|
|||
---
|
||||
title: "Microsoft Azure Files Storage"
|
||||
description: "Rclone docs for Microsoft Azure Files Storage"
|
||||
versionIntroduced: "v1.65"
|
||||
---
|
||||
|
||||
# Microsoft Azure File Storage
|
||||
# {{< icon "fab fa-windows" >}} Microsoft Azure Files Storage
|
||||
|
||||
Paths are specified as `remote:` You may put subdirectories in too,
|
||||
e.g. `remote:path/to/dir`.
|
||||
|
||||
## Configuration
|
||||
|
||||
Here is an example of making a Microsoft Azure Files Storage
|
||||
configuration. For a remote called `remote`. First run:
|
||||
|
||||
rclone config
|
||||
|
||||
This will guide you through an interactive setup process:
|
||||
|
||||
```
|
||||
No remotes found, make a new one?
|
||||
n) New remote
|
||||
s) Set configuration password
|
||||
q) Quit config
|
||||
n/s/q> n
|
||||
name> remote
|
||||
Type of storage to configure.
|
||||
Choose a number from below, or type in your own value
|
||||
[snip]
|
||||
XX / Microsoft Azure Files Storage
|
||||
\ "azurefiles"
|
||||
[snip]
|
||||
|
||||
Option account.
|
||||
Azure Storage Account Name.
|
||||
Set this to the Azure Storage Account Name in use.
|
||||
Leave blank to use SAS URL or connection string, otherwise it needs to be set.
|
||||
If this is blank and if env_auth is set it will be read from the
|
||||
environment variable `AZURE_STORAGE_ACCOUNT_NAME` if possible.
|
||||
Enter a value. Press Enter to leave empty.
|
||||
account> account_name
|
||||
|
||||
Option share_name.
|
||||
Azure Files Share Name.
|
||||
This is required and is the name of the share to access.
|
||||
Enter a value. Press Enter to leave empty.
|
||||
share_name> share_name
|
||||
|
||||
Option env_auth.
|
||||
Read credentials from runtime (environment variables, CLI or MSI).
|
||||
See the [authentication docs](/azurefiles#authentication) for full info.
|
||||
Enter a boolean value (true or false). Press Enter for the default (false).
|
||||
env_auth>
|
||||
|
||||
Option key.
|
||||
Storage Account Shared Key.
|
||||
Leave blank to use SAS URL or connection string.
|
||||
Enter a value. Press Enter to leave empty.
|
||||
key> base64encodedkey==
|
||||
|
||||
Option sas_url.
|
||||
SAS URL.
|
||||
Leave blank if using account/key or connection string.
|
||||
Enter a value. Press Enter to leave empty.
|
||||
sas_url>
|
||||
|
||||
Option connection_string.
|
||||
Azure Files Connection String.
|
||||
Enter a value. Press Enter to leave empty.
|
||||
connection_string>
|
||||
[snip]
|
||||
|
||||
Configuration complete.
|
||||
Options:
|
||||
- type: azurefiles
|
||||
- account: account_name
|
||||
- share_name: share_name
|
||||
- key: base64encodedkey==
|
||||
Keep this "remote" remote?
|
||||
y) Yes this is OK (default)
|
||||
e) Edit this remote
|
||||
d) Delete this remote
|
||||
y/e/d>
|
||||
```
|
||||
|
||||
Once configured you can use rclone.
|
||||
|
||||
See all files in the top level:
|
||||
|
||||
rclone lsf remote:
|
||||
|
||||
Make a new directory in the root:
|
||||
|
||||
rclone mkdir remote:dir
|
||||
|
||||
Recursively List the contents:
|
||||
|
||||
rclone ls remote:
|
||||
|
||||
Sync `/home/local/directory` to the remote directory, deleting any
|
||||
excess files in the directory.
|
||||
|
||||
rclone sync --interactive /home/local/directory remote:dir
|
||||
|
||||
### Modified time
|
||||
|
||||
Stored as azure standard `LastModified` time stored on files
|
||||
The modified time is stored as Azure standard `LastModified` time on
|
||||
files
|
||||
|
||||
### Performance
|
||||
|
||||
When uploading large files, increasing the value of
|
||||
`--azurefiles-upload-concurrency` will increase performance at the cost
|
||||
of using more memory. The default of 16 is set quite conservatively to
|
||||
use less memory. It maybe be necessary raise it to 64 or higher to
|
||||
fully utilize a 1 GBit/s link with a single file transfer.
|
||||
|
||||
### Restricted filename characters
|
||||
|
||||
In addition to the [default restricted characters set](/overview/#restricted-characters)
|
||||
the following characters are also replaced:
|
||||
|
||||
| Character | Value | Replacement |
|
||||
| --------- |:-----:|:-----------:|
|
||||
| " | 0x22 | ＂ |
|
||||
| * | 0x2A | ＊ |
|
||||
| : | 0x3A | ： |
|
||||
| < | 0x3C | ＜ |
|
||||
| > | 0x3E | ＞ |
|
||||
| ? | 0x3F | ？ |
|
||||
| \ | 0x5C | ＼ |
|
||||
| \| | 0x7C | ｜ |
|
||||
|
||||
File names can also not end with the following characters.
|
||||
These only get replaced if they are the last character in the name:
|
||||
|
||||
| Character | Value | Replacement |
|
||||
| --------- |:-----:|:-----------:|
|
||||
| . | 0x2E | ． |
|
||||
|
||||
Invalid UTF-8 bytes will also be [replaced](/overview/#invalid-utf8),
|
||||
as they can't be used in JSON strings.
|
||||
|
||||
### Hashes
|
||||
|
||||
MD5 hashes are stored with files.
|
||||
MD5 hashes are stored with files. Not all files will have MD5 hashes
|
||||
as these have to be uploaded with the file.
|
||||
|
||||
### Authentication {#authentication}
|
||||
|
||||
1. ConnectionString
|
||||
2. Accout and Key
|
||||
3. SAS URL
|
||||
There are a number of ways of supplying credentials for Azure Files
|
||||
Storage. Rclone tries them in the order of the sections below.
|
||||
|
||||
#### Env Auth
|
||||
|
||||
If the `env_auth` config parameter is `true` then rclone will pull
|
||||
credentials from the environment or runtime.
|
||||
|
||||
It tries these authentication methods in this order:
|
||||
|
||||
1. Environment Variables
|
||||
2. Managed Service Identity Credentials
|
||||
3. Azure CLI credentials (as used by the az tool)
|
||||
|
||||
These are described in the following sections
|
||||
|
||||
##### Env Auth: 1. Environment Variables
|
||||
|
||||
If `env_auth` is set and environment variables are present rclone
|
||||
authenticates a service principal with a secret or certificate, or a
|
||||
user with a password, depending on which environment variable are set.
|
||||
It reads configuration from these variables, in the following order:
|
||||
|
||||
1. Service principal with client secret
|
||||
- `AZURE_TENANT_ID`: ID of the service principal's tenant. Also called its "directory" ID.
|
||||
- `AZURE_CLIENT_ID`: the service principal's client ID
|
||||
- `AZURE_CLIENT_SECRET`: one of the service principal's client secrets
|
||||
2. Service principal with certificate
|
||||
- `AZURE_TENANT_ID`: ID of the service principal's tenant. Also called its "directory" ID.
|
||||
- `AZURE_CLIENT_ID`: the service principal's client ID
|
||||
- `AZURE_CLIENT_CERTIFICATE_PATH`: path to a PEM or PKCS12 certificate file including the private key.
|
||||
- `AZURE_CLIENT_CERTIFICATE_PASSWORD`: (optional) password for the certificate file.
|
||||
- `AZURE_CLIENT_SEND_CERTIFICATE_CHAIN`: (optional) Specifies whether an authentication request will include an x5c header to support subject name / issuer based authentication. When set to "true" or "1", authentication requests include the x5c header.
|
||||
3. User with username and password
|
||||
- `AZURE_TENANT_ID`: (optional) tenant to authenticate in. Defaults to "organizations".
|
||||
- `AZURE_CLIENT_ID`: client ID of the application the user will authenticate to
|
||||
- `AZURE_USERNAME`: a username (usually an email address)
|
||||
- `AZURE_PASSWORD`: the user's password
|
||||
4. Workload Identity
|
||||
- `AZURE_TENANT_ID`: Tenant to authenticate in.
|
||||
- `AZURE_CLIENT_ID`: Client ID of the application the user will authenticate to.
|
||||
- `AZURE_FEDERATED_TOKEN_FILE`: Path to projected service account token file.
|
||||
- `AZURE_AUTHORITY_HOST`: Authority of an Azure Active Directory endpoint (default: login.microsoftonline.com).
|
||||
|
||||
|
||||
##### Env Auth: 2. Managed Service Identity Credentials
|
||||
|
||||
When using Managed Service Identity if the VM(SS) on which this
|
||||
program is running has a system-assigned identity, it will be used by
|
||||
default. If the resource has no system-assigned but exactly one
|
||||
user-assigned identity, the user-assigned identity will be used by
|
||||
default.
|
||||
|
||||
If the resource has multiple user-assigned identities you will need to
|
||||
unset `env_auth` and set `use_msi` instead. See the [`use_msi`
|
||||
section](#use_msi).
|
||||
|
||||
##### Env Auth: 3. Azure CLI credentials (as used by the az tool)
|
||||
|
||||
Credentials created with the `az` tool can be picked up using `env_auth`.
|
||||
|
||||
For example if you were to login with a service principal like this:
|
||||
|
||||
az login --service-principal -u XXX -p XXX --tenant XXX
|
||||
|
||||
Then you could access rclone resources like this:
|
||||
|
||||
rclone lsf :azurefiles,env_auth,account=ACCOUNT:
|
||||
|
||||
Or
|
||||
|
||||
rclone lsf --azurefiles-env-auth --azurefiles-account=ACCOUNT :azurefiles:
|
||||
|
||||
#### Account and Shared Key
|
||||
|
||||
This is the most straight forward and least flexible way. Just fill
|
||||
in the `account` and `key` lines and leave the rest blank.
|
||||
|
||||
#### SAS URL
|
||||
|
||||
To use it leave `account`, `key` and `connection_string` blank and fill in `sas_url`.
|
||||
|
||||
#### Connection String
|
||||
|
||||
To use it leave `account`, `key` and "sas_url" blank and fill in `connection_string`.
|
||||
|
||||
#### Service principal with client secret
|
||||
|
||||
If these variables are set, rclone will authenticate with a service principal with a client secret.
|
||||
|
||||
- `tenant`: ID of the service principal's tenant. Also called its "directory" ID.
|
||||
- `client_id`: the service principal's client ID
|
||||
- `client_secret`: one of the service principal's client secrets
|
||||
|
||||
The credentials can also be placed in a file using the
|
||||
`service_principal_file` configuration option.
|
||||
|
||||
#### Service principal with certificate
|
||||
|
||||
If these variables are set, rclone will authenticate with a service principal with certificate.
|
||||
|
||||
- `tenant`: ID of the service principal's tenant. Also called its "directory" ID.
|
||||
- `client_id`: the service principal's client ID
|
||||
- `client_certificate_path`: path to a PEM or PKCS12 certificate file including the private key.
|
||||
- `client_certificate_password`: (optional) password for the certificate file.
|
||||
- `client_send_certificate_chain`: (optional) Specifies whether an authentication request will include an x5c header to support subject name / issuer based authentication. When set to "true" or "1", authentication requests include the x5c header.
|
||||
|
||||
**NB** `client_certificate_password` must be obscured - see [rclone obscure](/commands/rclone_obscure/).
|
||||
|
||||
#### User with username and password
|
||||
|
||||
If these variables are set, rclone will authenticate with username and password.
|
||||
|
||||
- `tenant`: (optional) tenant to authenticate in. Defaults to "organizations".
|
||||
- `client_id`: client ID of the application the user will authenticate to
|
||||
- `username`: a username (usually an email address)
|
||||
- `password`: the user's password
|
||||
|
||||
Microsoft doesn't recommend this kind of authentication, because it's
|
||||
less secure than other authentication flows. This method is not
|
||||
interactive, so it isn't compatible with any form of multi-factor
|
||||
authentication, and the application must already have user or admin
|
||||
consent. This credential can only authenticate work and school
|
||||
accounts; it can't authenticate Microsoft accounts.
|
||||
|
||||
**NB** `password` must be obscured - see [rclone obscure](/commands/rclone_obscure/).
|
||||
|
||||
#### Managed Service Identity Credentials {#use_msi}
|
||||
|
||||
If `use_msi` is set then managed service identity credentials are
|
||||
used. This authentication only works when running in an Azure service.
|
||||
`env_auth` needs to be unset to use this.
|
||||
|
||||
However if you have multiple user identities to choose from these must
|
||||
be explicitly specified using exactly one of the `msi_object_id`,
|
||||
`msi_client_id`, or `msi_mi_res_id` parameters.
|
||||
|
||||
If none of `msi_object_id`, `msi_client_id`, or `msi_mi_res_id` is
|
||||
set, this is is equivalent to using `env_auth`.
|
||||
|
||||
{{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/azurefiles/azurefiles.go then run make backenddocs" >}}
|
||||
### Standard options
|
||||
|
||||
Here are the Standard options specific to azurefiles (Microsoft Azure Files).
|
||||
|
||||
#### --azurefiles-account
|
||||
|
||||
Azure Storage Account Name.
|
||||
|
||||
Set this to the Azure Storage Account Name in use.
|
||||
|
||||
Leave blank to use SAS URL or connection string, otherwise it needs to be set.
|
||||
|
||||
If this is blank and if env_auth is set it will be read from the
|
||||
environment variable `AZURE_STORAGE_ACCOUNT_NAME` if possible.
|
||||
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: account
|
||||
- Env Var: RCLONE_AZUREFILES_ACCOUNT
|
||||
- Type: string
|
||||
- Required: false
|
||||
|
||||
#### --azurefiles-share-name
|
||||
|
||||
Azure Files Share Name.
|
||||
|
||||
This is required and is the name of the share to access.
|
||||
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: share_name
|
||||
- Env Var: RCLONE_AZUREFILES_SHARE_NAME
|
||||
- Type: string
|
||||
- Required: false
|
||||
|
||||
#### --azurefiles-env-auth
|
||||
|
||||
Read credentials from runtime (environment variables, CLI or MSI).
|
||||
|
||||
See the [authentication docs](/azurefiles#authentication) for full info.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: env_auth
|
||||
- Env Var: RCLONE_AZUREFILES_ENV_AUTH
|
||||
- Type: bool
|
||||
- Default: false
|
||||
|
||||
#### --azurefiles-key
|
||||
|
||||
Storage Account Shared Key.
|
||||
|
||||
Leave blank to use SAS URL or connection string.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: key
|
||||
- Env Var: RCLONE_AZUREFILES_KEY
|
||||
- Type: string
|
||||
- Required: false
|
||||
|
||||
#### --azurefiles-sas-url
|
||||
|
||||
SAS URL.
|
||||
|
||||
Leave blank if using account/key or connection string.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: sas_url
|
||||
- Env Var: RCLONE_AZUREFILES_SAS_URL
|
||||
- Type: string
|
||||
- Required: false
|
||||
|
||||
#### --azurefiles-connection-string
|
||||
|
||||
Azure Files Connection String.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: connection_string
|
||||
- Env Var: RCLONE_AZUREFILES_CONNECTION_STRING
|
||||
- Type: string
|
||||
- Required: false
|
||||
|
||||
#### --azurefiles-tenant
|
||||
|
||||
ID of the service principal's tenant. Also called its directory ID.
|
||||
|
||||
Set this if using
|
||||
- Service principal with client secret
|
||||
- Service principal with certificate
|
||||
- User with username and password
|
||||
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: tenant
|
||||
- Env Var: RCLONE_AZUREFILES_TENANT
|
||||
- Type: string
|
||||
- Required: false
|
||||
|
||||
#### --azurefiles-client-id
|
||||
|
||||
The ID of the client in use.
|
||||
|
||||
Set this if using
|
||||
- Service principal with client secret
|
||||
- Service principal with certificate
|
||||
- User with username and password
|
||||
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_id
|
||||
- Env Var: RCLONE_AZUREFILES_CLIENT_ID
|
||||
- Type: string
|
||||
- Required: false
|
||||
|
||||
#### --azurefiles-client-secret
|
||||
|
||||
One of the service principal's client secrets
|
||||
|
||||
Set this if using
|
||||
- Service principal with client secret
|
||||
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_secret
|
||||
- Env Var: RCLONE_AZUREFILES_CLIENT_SECRET
|
||||
- Type: string
|
||||
- Required: false
|
||||
|
||||
#### --azurefiles-client-certificate-path
|
||||
|
||||
Path to a PEM or PKCS12 certificate file including the private key.
|
||||
|
||||
Set this if using
|
||||
- Service principal with certificate
|
||||
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_certificate_path
|
||||
- Env Var: RCLONE_AZUREFILES_CLIENT_CERTIFICATE_PATH
|
||||
- Type: string
|
||||
- Required: false
|
||||
|
||||
#### --azurefiles-client-certificate-password
|
||||
|
||||
Password for the certificate file (optional).
|
||||
|
||||
Optionally set this if using
|
||||
- Service principal with certificate
|
||||
|
||||
And the certificate has a password.
|
||||
|
||||
|
||||
**NB** Input to this must be obscured - see [rclone obscure](/commands/rclone_obscure/).
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_certificate_password
|
||||
- Env Var: RCLONE_AZUREFILES_CLIENT_CERTIFICATE_PASSWORD
|
||||
- Type: string
|
||||
- Required: false
|
||||
|
||||
### Advanced options
|
||||
|
||||
Here are the Advanced options specific to azurefiles (Microsoft Azure Files).
|
||||
|
||||
#### --azurefiles-client-send-certificate-chain
|
||||
|
||||
Send the certificate chain when using certificate auth.
|
||||
|
||||
Specifies whether an authentication request will include an x5c header
|
||||
to support subject name / issuer based authentication. When set to
|
||||
true, authentication requests include the x5c header.
|
||||
|
||||
Optionally set this if using
|
||||
- Service principal with certificate
|
||||
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_send_certificate_chain
|
||||
- Env Var: RCLONE_AZUREFILES_CLIENT_SEND_CERTIFICATE_CHAIN
|
||||
- Type: bool
|
||||
- Default: false
|
||||
|
||||
#### --azurefiles-username
|
||||
|
||||
User name (usually an email address)
|
||||
|
||||
Set this if using
|
||||
- User with username and password
|
||||
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: username
|
||||
- Env Var: RCLONE_AZUREFILES_USERNAME
|
||||
- Type: string
|
||||
- Required: false
|
||||
|
||||
#### --azurefiles-password
|
||||
|
||||
The user's password
|
||||
|
||||
Set this if using
|
||||
- User with username and password
|
||||
|
||||
|
||||
**NB** Input to this must be obscured - see [rclone obscure](/commands/rclone_obscure/).
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: password
|
||||
- Env Var: RCLONE_AZUREFILES_PASSWORD
|
||||
- Type: string
|
||||
- Required: false
|
||||
|
||||
#### --azurefiles-service-principal-file
|
||||
|
||||
Path to file containing credentials for use with a service principal.
|
||||
|
||||
Leave blank normally. Needed only if you want to use a service principal instead of interactive login.
|
||||
|
||||
$ az ad sp create-for-rbac --name "<name>" \
|
||||
--role "Storage Files Data Owner" \
|
||||
--scopes "/subscriptions/<subscription>/resourceGroups/<resource-group>/providers/Microsoft.Storage/storageAccounts/<storage-account>/blobServices/default/containers/<container>" \
|
||||
> azure-principal.json
|
||||
|
||||
See ["Create an Azure service principal"](https://docs.microsoft.com/en-us/cli/azure/create-an-azure-service-principal-azure-cli) and ["Assign an Azure role for access to files data"](https://docs.microsoft.com/en-us/azure/storage/common/storage-auth-aad-rbac-cli) pages for more details.
|
||||
|
||||
**NB** this section needs updating for Azure Files - pull requests appreciated!
|
||||
|
||||
It may be more convenient to put the credentials directly into the
|
||||
rclone config file under the `client_id`, `tenant` and `client_secret`
|
||||
keys instead of setting `service_principal_file`.
|
||||
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: service_principal_file
|
||||
- Env Var: RCLONE_AZUREFILES_SERVICE_PRINCIPAL_FILE
|
||||
- Type: string
|
||||
- Required: false
|
||||
|
||||
#### --azurefiles-use-msi
|
||||
|
||||
Use a managed service identity to authenticate (only works in Azure).
|
||||
|
||||
When true, use a [managed service identity](https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/)
|
||||
to authenticate to Azure Storage instead of a SAS token or account key.
|
||||
|
||||
If the VM(SS) on which this program is running has a system-assigned identity, it will
|
||||
be used by default. If the resource has no system-assigned but exactly one user-assigned identity,
|
||||
the user-assigned identity will be used by default. If the resource has multiple user-assigned
|
||||
identities, the identity to use must be explicitly specified using exactly one of the msi_object_id,
|
||||
msi_client_id, or msi_mi_res_id parameters.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: use_msi
|
||||
- Env Var: RCLONE_AZUREFILES_USE_MSI
|
||||
- Type: bool
|
||||
- Default: false
|
||||
|
||||
#### --azurefiles-msi-object-id
|
||||
|
||||
Object ID of the user-assigned MSI to use, if any.
|
||||
|
||||
Leave blank if msi_client_id or msi_mi_res_id specified.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: msi_object_id
|
||||
- Env Var: RCLONE_AZUREFILES_MSI_OBJECT_ID
|
||||
- Type: string
|
||||
- Required: false
|
||||
|
||||
#### --azurefiles-msi-client-id
|
||||
|
||||
Object ID of the user-assigned MSI to use, if any.
|
||||
|
||||
Leave blank if msi_object_id or msi_mi_res_id specified.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: msi_client_id
|
||||
- Env Var: RCLONE_AZUREFILES_MSI_CLIENT_ID
|
||||
- Type: string
|
||||
- Required: false
|
||||
|
||||
#### --azurefiles-msi-mi-res-id
|
||||
|
||||
Azure resource ID of the user-assigned MSI to use, if any.
|
||||
|
||||
Leave blank if msi_client_id or msi_object_id specified.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: msi_mi_res_id
|
||||
- Env Var: RCLONE_AZUREFILES_MSI_MI_RES_ID
|
||||
- Type: string
|
||||
- Required: false
|
||||
|
||||
#### --azurefiles-endpoint
|
||||
|
||||
Endpoint for the service.
|
||||
|
||||
Leave blank normally.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: endpoint
|
||||
- Env Var: RCLONE_AZUREFILES_ENDPOINT
|
||||
- Type: string
|
||||
- Required: false
|
||||
|
||||
#### --azurefiles-chunk-size
|
||||
|
||||
Upload chunk size.
|
||||
|
||||
Note that this is stored in memory and there may be up to
|
||||
"--transfers" * "--azurefile-upload-concurrency" chunks stored at once
|
||||
in memory.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: chunk_size
|
||||
- Env Var: RCLONE_AZUREFILES_CHUNK_SIZE
|
||||
- Type: SizeSuffix
|
||||
- Default: 4Mi
|
||||
|
||||
#### --azurefiles-upload-concurrency
|
||||
|
||||
Concurrency for multipart uploads.
|
||||
|
||||
This is the number of chunks of the same file that are uploaded
|
||||
concurrently.
|
||||
|
||||
If you are uploading small numbers of large files over high-speed
|
||||
links and these uploads do not fully utilize your bandwidth, then
|
||||
increasing this may help to speed up the transfers.
|
||||
|
||||
Note that chunks are stored in memory and there may be up to
|
||||
"--transfers" * "--azurefile-upload-concurrency" chunks stored at once
|
||||
in memory.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: upload_concurrency
|
||||
- Env Var: RCLONE_AZUREFILES_UPLOAD_CONCURRENCY
|
||||
- Type: int
|
||||
- Default: 16
|
||||
|
||||
#### --azurefiles-max-stream-size
|
||||
|
||||
Max size for streamed files.
|
||||
|
||||
Azure files needs to know in advance how big the file will be. When
|
||||
rclone doesn't know it uses this value instead.
|
||||
|
||||
This will be used when rclone is streaming data, the most common uses are:
|
||||
|
||||
- Uploading files with `--vfs-cache-mode off` with `rclone mount`
|
||||
- Using `rclone rcat`
|
||||
- Copying files with unknown length
|
||||
|
||||
You will need this much free space in the share as the file will be this size temporarily.
|
||||
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: max_stream_size
|
||||
- Env Var: RCLONE_AZUREFILES_MAX_STREAM_SIZE
|
||||
- Type: SizeSuffix
|
||||
- Default: 10Gi
|
||||
|
||||
#### --azurefiles-encoding
|
||||
|
||||
The encoding for the backend.
|
||||
|
||||
See the [encoding section in the overview](/overview/#encoding) for more info.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: encoding
|
||||
- Env Var: RCLONE_AZUREFILES_ENCODING
|
||||
- Type: Encoding
|
||||
- Default: Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Del,Ctl,RightPeriod,InvalidUtf8,Dot
|
||||
|
||||
{{< rem autogenerated options stop >}}
|
||||
|
||||
### Custom upload headers
|
||||
|
||||
You can set custom upload headers with the `--header-upload` flag.
|
||||
|
||||
- Cache-Control
|
||||
- Content-Disposition
|
||||
- Content-Encoding
|
||||
- Content-Language
|
||||
- Content-Type
|
||||
|
||||
Eg `--header-upload "Content-Type: text/potato"`
|
||||
|
||||
## Limitations
|
||||
|
||||
MD5 sums are only uploaded with chunked files if the source has an MD5
|
||||
sum. This will always be the case for a local to azure copy.
|
||||
|
|
|
|||
|
|
@ -58,6 +58,7 @@ See the following for detailed instructions for
|
|||
* [Mega](/mega/)
|
||||
* [Memory](/memory/)
|
||||
* [Microsoft Azure Blob Storage](/azureblob/)
|
||||
* [Microsoft Azure Files Storage](/azurefiles/)
|
||||
* [Microsoft OneDrive](/onedrive/)
|
||||
* [OpenStack Swift / Rackspace Cloudfiles / Blomp Cloud Storage / Memset Memstore](/swift/)
|
||||
* [OpenDrive](/opendrive/)
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ Here is an overview of the major features of each cloud storage system.
|
|||
| Mega | - | - | No | Yes | - | - |
|
||||
| Memory | MD5 | R/W | No | No | - | - |
|
||||
| Microsoft Azure Blob Storage | MD5 | R/W | No | No | R/W | - |
|
||||
| Microsoft Azure Files Storage | MD5 | R/W | Yes | No | R/W | - |
|
||||
| Microsoft OneDrive | QuickXorHash ⁵ | R/W | Yes | No | R | - |
|
||||
| OpenDrive | MD5 | R/W | Yes | Partial ⁸ | - | - |
|
||||
| OpenStack Swift | MD5 | R/W | No | No | R/W | - |
|
||||
|
|
@ -490,6 +491,7 @@ upon backend-specific capabilities.
|
|||
| Mega | Yes | No | Yes | Yes | Yes | No | No | No | Yes | Yes | Yes |
|
||||
| Memory | No | Yes | No | No | No | Yes | Yes | No | No | No | No |
|
||||
| Microsoft Azure Blob Storage | Yes | Yes | No | No | No | Yes | Yes | Yes | No | No | No |
|
||||
| Microsoft Azure Files Storage | No | Yes | Yes | Yes | No | No | Yes | Yes | No | Yes | Yes |
|
||||
| Microsoft OneDrive | Yes | Yes | Yes | Yes | Yes | Yes | No | No | Yes | Yes | Yes |
|
||||
| OpenDrive | Yes | Yes | Yes | Yes | No | No | No | No | No | No | Yes |
|
||||
| OpenStack Swift | Yes ¹ | Yes | No | No | No | Yes | Yes | No | No | Yes | No |
|
||||
|
|
|
|||
|
|
@ -81,6 +81,7 @@
|
|||
<a class="dropdown-item" href="/mega/"><i class="fa fa-archive fa-fw"></i> Mega</a>
|
||||
<a class="dropdown-item" href="/memory/"><i class="fas fa-memory fa-fw"></i> Memory</a>
|
||||
<a class="dropdown-item" href="/azureblob/"><i class="fab fa-windows fa-fw"></i> Microsoft Azure Blob Storage</a>
|
||||
<a class="dropdown-item" href="/azurefiles/"><i class="fab fa-windows fa-fw"></i> Microsoft Azure Files Storage</a>
|
||||
<a class="dropdown-item" href="/onedrive/"><i class="fab fa-windows fa-fw"></i> Microsoft OneDrive</a>
|
||||
<a class="dropdown-item" href="/opendrive/"><i class="fa fa-space-shuttle fa-fw"></i> OpenDrive</a>
|
||||
<a class="dropdown-item" href="/qingstor/"><i class="fas fa-hdd fa-fw"></i> QingStor</a>
|
||||
|
|
|
|||
|
|
@ -320,6 +320,8 @@ backends:
|
|||
- backend: "azureblob"
|
||||
remote: "TestAzureBlob,directory_markers:"
|
||||
fastlist: true
|
||||
- backend: "azurefiles"
|
||||
remote: "TestAzureFiles:"
|
||||
- backend: "pcloud"
|
||||
remote: "TestPcloud:"
|
||||
fastlist: true
|
||||
|
|
|
|||
Loading…
Reference in New Issue