From 80bccacd83be78bf72649f48069f6260c3895309 Mon Sep 17 00:00:00 2001 From: Ivan Andreev Date: Mon, 14 Jun 2021 14:42:49 +0300 Subject: [PATCH] fs: split overgrown fs.go (#5405) Nothing is added or removed and no package is renamed by this change. Just rearrange definitions between source files in the fs directory. New source files: - types.go Filesystem types and interfaces - features.go Features and optional interfaces - registry.go Filesystem registry and backend options - newfs.go NewFs and its helpers - configmap.go Getters and Setters for ConfigMap - pacer.go Pacer with logging and calculator The final fs.go contains what is left. Also rename options.go to open_options.go to dissociate from registry options. --- fs/configmap.go | 128 ++ fs/features.go | 691 ++++++++ fs/fs.go | 1513 ------------------ fs/newfs.go | 122 ++ fs/{options.go => open_options.go} | 2 +- fs/{options_test.go => open_options_test.go} | 0 fs/pacer.go | 94 ++ fs/registry.go | 277 ++++ fs/types.go | 266 +++ 9 files changed, 1579 insertions(+), 1514 deletions(-) create mode 100644 fs/configmap.go create mode 100644 fs/features.go create mode 100644 fs/newfs.go rename fs/{options.go => open_options.go} (99%) rename fs/{options_test.go => open_options_test.go} (100%) create mode 100644 fs/pacer.go create mode 100644 fs/registry.go create mode 100644 fs/types.go diff --git a/fs/configmap.go b/fs/configmap.go new file mode 100644 index 000000000..db2ca6aff --- /dev/null +++ b/fs/configmap.go @@ -0,0 +1,128 @@ +// Getters and Setters for ConfigMap + +package fs + +import ( + "os" + "strings" + + "github.com/rclone/rclone/fs/config/configmap" +) + +// A configmap.Getter to read from the environment RCLONE_CONFIG_backend_option_name +type configEnvVars string + +// Get a config item from the environment variables if possible +func (configName configEnvVars) Get(key string) (value string, ok bool) { + return os.LookupEnv(ConfigToEnv(string(configName), key)) +} + +// A configmap.Getter to read from the environment RCLONE_option_name +type optionEnvVars struct { + fsInfo *RegInfo +} + +// Get a config item from the option environment variables if possible +func (oev optionEnvVars) Get(key string) (value string, ok bool) { + opt := oev.fsInfo.Options.Get(key) + if opt == nil { + return "", false + } + // For options with NoPrefix set, check without prefix too + if opt.NoPrefix { + value, ok = os.LookupEnv(OptionToEnv(key)) + if ok { + return value, ok + } + } + return os.LookupEnv(OptionToEnv(oev.fsInfo.Prefix + "-" + key)) +} + +// A configmap.Getter to read either the default value or the set +// value from the RegInfo.Options +type regInfoValues struct { + fsInfo *RegInfo + useDefault bool +} + +// override the values in configMap with the either the flag values or +// the default values +func (r *regInfoValues) Get(key string) (value string, ok bool) { + opt := r.fsInfo.Options.Get(key) + if opt != nil && (r.useDefault || opt.Value != nil) { + return opt.String(), true + } + return "", false +} + +// A configmap.Setter to read from the config file +type setConfigFile string + +// Set a config item into the config file +func (section setConfigFile) Set(key, value string) { + if strings.HasPrefix(string(section), ":") { + Logf(nil, "Can't save config %q = %q for on the fly backend %q", key, value, section) + return + } + Debugf(nil, "Saving config %q = %q in section %q of the config file", key, value, section) + err := ConfigFileSet(string(section), key, value) + if err != nil { + Errorf(nil, "Failed saving config %q = %q in section %q of the config file: %v", key, value, section, err) + } +} + +// A configmap.Getter to read from the config file +type getConfigFile string + +// Get a config item from the config file +func (section getConfigFile) Get(key string) (value string, ok bool) { + value, ok = ConfigFileGet(string(section), key) + // Ignore empty lines in the config file + if value == "" { + ok = false + } + return value, ok +} + +// ConfigMap creates a configmap.Map from the *RegInfo and the +// configName passed in. If connectionStringConfig has any entries (it may be nil), +// then it will be added to the lookup with the highest priority. +// +// If fsInfo is nil then the returned configmap.Map should only be +// used for reading non backend specific parameters, such as "type". +func ConfigMap(fsInfo *RegInfo, configName string, connectionStringConfig configmap.Simple) (config *configmap.Map) { + // Create the config + config = configmap.New() + + // Read the config, more specific to least specific + + // Config from connection string + if len(connectionStringConfig) > 0 { + config.AddGetter(connectionStringConfig, configmap.PriorityNormal) + } + + // flag values + if fsInfo != nil { + config.AddGetter(®InfoValues{fsInfo, false}, configmap.PriorityNormal) + } + + // remote specific environment vars + config.AddGetter(configEnvVars(configName), configmap.PriorityNormal) + + // backend specific environment vars + if fsInfo != nil { + config.AddGetter(optionEnvVars{fsInfo: fsInfo}, configmap.PriorityNormal) + } + + // config file + config.AddGetter(getConfigFile(configName), configmap.PriorityConfig) + + // default values + if fsInfo != nil { + config.AddGetter(®InfoValues{fsInfo, true}, configmap.PriorityDefault) + } + + // Set Config + config.AddSetter(setConfigFile(configName)) + return config +} diff --git a/fs/features.go b/fs/features.go new file mode 100644 index 000000000..6db6bd548 --- /dev/null +++ b/fs/features.go @@ -0,0 +1,691 @@ +// Filesystem features and optional interfaces + +package fs + +import ( + "context" + "io" + "reflect" + "strings" + "time" +) + +// Features describe the optional features of the Fs +type Features struct { + // Feature flags, whether Fs + CaseInsensitive bool // has case insensitive files + DuplicateFiles bool // allows duplicate files + ReadMimeType bool // can read the mime type of objects + WriteMimeType bool // can set the mime type of objects + CanHaveEmptyDirectories bool // can have empty directories + BucketBased bool // is bucket based (like s3, swift, etc.) + BucketBasedRootOK bool // is bucket based and can use from root + SetTier bool // allows set tier functionality on objects + GetTier bool // allows to retrieve storage tier of objects + ServerSideAcrossConfigs bool // can server-side copy between different remotes of the same type + IsLocal bool // is the local backend + SlowModTime bool // if calling ModTime() generally takes an extra transaction + SlowHash bool // if calling Hash() generally takes an extra transaction + + // Purge all files in the directory specified + // + // Implement this if you have a way of deleting all the files + // quicker than just running Remove() on the result of List() + // + // Return an error if it doesn't exist + Purge func(ctx context.Context, dir string) error + + // Copy src to this remote using server-side copy operations. + // + // This is stored with the remote path given + // + // It returns the destination Object and a possible error + // + // Will only be called if src.Fs().Name() == f.Name() + // + // If it isn't possible then return fs.ErrorCantCopy + Copy func(ctx context.Context, src Object, remote string) (Object, error) + + // Move src to this remote using server-side move operations. + // + // This is stored with the remote path given + // + // It returns the destination Object and a possible error + // + // Will only be called if src.Fs().Name() == f.Name() + // + // If it isn't possible then return fs.ErrorCantMove + Move func(ctx context.Context, src Object, remote string) (Object, error) + + // DirMove moves src, srcRemote to this remote at dstRemote + // using server-side move operations. + // + // Will only be called if src.Fs().Name() == f.Name() + // + // If it isn't possible then return fs.ErrorCantDirMove + // + // If destination exists then return fs.ErrorDirExists + DirMove func(ctx context.Context, src Fs, srcRemote, dstRemote string) error + + // ChangeNotify calls the passed function with a path + // that has had changes. If the implementation + // uses polling, it should adhere to the given interval. + ChangeNotify func(context.Context, func(string, EntryType), <-chan time.Duration) + + // UnWrap returns the Fs that this Fs is wrapping + UnWrap func() Fs + + // WrapFs returns the Fs that is wrapping this Fs + WrapFs func() Fs + + // SetWrapper sets the Fs that is wrapping this Fs + SetWrapper func(f Fs) + + // DirCacheFlush resets the directory cache - used in testing + // as an optional interface + DirCacheFlush func() + + // PublicLink generates a public link to the remote path (usually readable by anyone) + PublicLink func(ctx context.Context, remote string, expire Duration, unlink bool) (string, error) + + // Put in to the remote path with the modTime given of the given size + // + // May create the object even if it returns an error - if so + // will return the object and the error, otherwise will return + // nil and the error + // + // May create duplicates or return errors if src already + // exists. + PutUnchecked func(ctx context.Context, in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error) + + // PutStream uploads to the remote path with the modTime given of indeterminate size + // + // May create the object even if it returns an error - if so + // will return the object and the error, otherwise will return + // nil and the error + PutStream func(ctx context.Context, in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error) + + // MergeDirs merges the contents of all the directories passed + // in into the first one and rmdirs the other directories. + MergeDirs func(ctx context.Context, dirs []Directory) error + + // CleanUp the trash in the Fs + // + // Implement this if you have a way of emptying the trash or + // otherwise cleaning up old versions of files. + CleanUp func(ctx context.Context) error + + // ListR lists the objects and directories of the Fs starting + // from dir recursively into out. + // + // dir should be "" to start from the root, and should not + // have trailing slashes. + // + // This should return ErrDirNotFound if the directory isn't + // found. + // + // It should call callback for each tranche of entries read. + // These need not be returned in any particular order. If + // callback returns an error then the listing will stop + // immediately. + // + // Don't implement this unless you have a more efficient way + // of listing recursively that doing a directory traversal. + ListR ListRFn + + // About gets quota information from the Fs + About func(ctx context.Context) (*Usage, error) + + // OpenWriterAt opens with a handle for random access writes + // + // Pass in the remote desired and the size if known. + // + // It truncates any existing object + OpenWriterAt func(ctx context.Context, remote string, size int64) (WriterAtCloser, error) + + // UserInfo returns info about the connected user + UserInfo func(ctx context.Context) (map[string]string, error) + + // Disconnect the current user + Disconnect func(ctx context.Context) error + + // Command the backend to run a named command + // + // The command run is name + // args may be used to read arguments from + // opts may be used to read optional arguments from + // + // The result should be capable of being JSON encoded + // If it is a string or a []string it will be shown to the user + // otherwise it will be JSON encoded and shown to the user like that + Command func(ctx context.Context, name string, arg []string, opt map[string]string) (interface{}, error) + + // Shutdown the backend, closing any background tasks and any + // cached connections. + Shutdown func(ctx context.Context) error +} + +// Disable nil's out the named feature. If it isn't found then it +// will log a message. +func (ft *Features) Disable(name string) *Features { + v := reflect.ValueOf(ft).Elem() + vType := v.Type() + for i := 0; i < v.NumField(); i++ { + vName := vType.Field(i).Name + field := v.Field(i) + if strings.EqualFold(name, vName) { + if !field.CanSet() { + Errorf(nil, "Can't set Feature %q", name) + } else { + zero := reflect.Zero(field.Type()) + field.Set(zero) + Debugf(nil, "Reset feature %q", name) + } + } + } + return ft +} + +// List returns a slice of all the possible feature names +func (ft *Features) List() (out []string) { + v := reflect.ValueOf(ft).Elem() + vType := v.Type() + for i := 0; i < v.NumField(); i++ { + out = append(out, vType.Field(i).Name) + } + return out +} + +// Enabled returns a map of features with keys showing whether they +// are enabled or not +func (ft *Features) Enabled() (features map[string]bool) { + v := reflect.ValueOf(ft).Elem() + vType := v.Type() + features = make(map[string]bool, v.NumField()) + for i := 0; i < v.NumField(); i++ { + vName := vType.Field(i).Name + field := v.Field(i) + if field.Kind() == reflect.Func { + // Can't compare functions + features[vName] = !field.IsNil() + } else { + zero := reflect.Zero(field.Type()) + features[vName] = field.Interface() != zero.Interface() + } + } + return features +} + +// DisableList nil's out the comma separated list of named features. +// If it isn't found then it will log a message. +func (ft *Features) DisableList(list []string) *Features { + for _, feature := range list { + ft.Disable(strings.TrimSpace(feature)) + } + return ft +} + +// Fill fills in the function pointers in the Features struct from the +// optional interfaces. It returns the original updated Features +// struct passed in. +func (ft *Features) Fill(ctx context.Context, f Fs) *Features { + if do, ok := f.(Purger); ok { + ft.Purge = do.Purge + } + if do, ok := f.(Copier); ok { + ft.Copy = do.Copy + } + if do, ok := f.(Mover); ok { + ft.Move = do.Move + } + if do, ok := f.(DirMover); ok { + ft.DirMove = do.DirMove + } + if do, ok := f.(ChangeNotifier); ok { + ft.ChangeNotify = do.ChangeNotify + } + if do, ok := f.(UnWrapper); ok { + ft.UnWrap = do.UnWrap + } + if do, ok := f.(Wrapper); ok { + ft.WrapFs = do.WrapFs + ft.SetWrapper = do.SetWrapper + } + if do, ok := f.(DirCacheFlusher); ok { + ft.DirCacheFlush = do.DirCacheFlush + } + if do, ok := f.(PublicLinker); ok { + ft.PublicLink = do.PublicLink + } + if do, ok := f.(PutUncheckeder); ok { + ft.PutUnchecked = do.PutUnchecked + } + if do, ok := f.(PutStreamer); ok { + ft.PutStream = do.PutStream + } + if do, ok := f.(MergeDirser); ok { + ft.MergeDirs = do.MergeDirs + } + if do, ok := f.(CleanUpper); ok { + ft.CleanUp = do.CleanUp + } + if do, ok := f.(ListRer); ok { + ft.ListR = do.ListR + } + if do, ok := f.(Abouter); ok { + ft.About = do.About + } + if do, ok := f.(OpenWriterAter); ok { + ft.OpenWriterAt = do.OpenWriterAt + } + if do, ok := f.(UserInfoer); ok { + ft.UserInfo = do.UserInfo + } + if do, ok := f.(Disconnecter); ok { + ft.Disconnect = do.Disconnect + } + if do, ok := f.(Commander); ok { + ft.Command = do.Command + } + if do, ok := f.(Shutdowner); ok { + ft.Shutdown = do.Shutdown + } + return ft.DisableList(GetConfig(ctx).DisableFeatures) +} + +// Mask the Features with the Fs passed in +// +// Only optional features which are implemented in both the original +// Fs AND the one passed in will be advertised. Any features which +// aren't in both will be set to false/nil, except for UnWrap/Wrap which +// will be left untouched. +func (ft *Features) Mask(ctx context.Context, f Fs) *Features { + mask := f.Features() + ft.CaseInsensitive = ft.CaseInsensitive && mask.CaseInsensitive + ft.DuplicateFiles = ft.DuplicateFiles && mask.DuplicateFiles + ft.ReadMimeType = ft.ReadMimeType && mask.ReadMimeType + ft.WriteMimeType = ft.WriteMimeType && mask.WriteMimeType + ft.CanHaveEmptyDirectories = ft.CanHaveEmptyDirectories && mask.CanHaveEmptyDirectories + ft.BucketBased = ft.BucketBased && mask.BucketBased + ft.BucketBasedRootOK = ft.BucketBasedRootOK && mask.BucketBasedRootOK + ft.SetTier = ft.SetTier && mask.SetTier + ft.GetTier = ft.GetTier && mask.GetTier + ft.ServerSideAcrossConfigs = ft.ServerSideAcrossConfigs && mask.ServerSideAcrossConfigs + // ft.IsLocal = ft.IsLocal && mask.IsLocal Don't propagate IsLocal + ft.SlowModTime = ft.SlowModTime && mask.SlowModTime + ft.SlowHash = ft.SlowHash && mask.SlowHash + + if mask.Purge == nil { + ft.Purge = nil + } + if mask.Copy == nil { + ft.Copy = nil + } + if mask.Move == nil { + ft.Move = nil + } + if mask.DirMove == nil { + ft.DirMove = nil + } + if mask.ChangeNotify == nil { + ft.ChangeNotify = nil + } + // if mask.UnWrap == nil { + // ft.UnWrap = nil + // } + // if mask.Wrapper == nil { + // ft.Wrapper = nil + // } + if mask.DirCacheFlush == nil { + ft.DirCacheFlush = nil + } + if mask.PublicLink == nil { + ft.PublicLink = nil + } + if mask.PutUnchecked == nil { + ft.PutUnchecked = nil + } + if mask.PutStream == nil { + ft.PutStream = nil + } + if mask.MergeDirs == nil { + ft.MergeDirs = nil + } + if mask.CleanUp == nil { + ft.CleanUp = nil + } + if mask.ListR == nil { + ft.ListR = nil + } + if mask.About == nil { + ft.About = nil + } + if mask.OpenWriterAt == nil { + ft.OpenWriterAt = nil + } + if mask.UserInfo == nil { + ft.UserInfo = nil + } + if mask.Disconnect == nil { + ft.Disconnect = nil + } + // Command is always local so we don't mask it + if mask.Shutdown == nil { + ft.Shutdown = nil + } + return ft.DisableList(GetConfig(ctx).DisableFeatures) +} + +// Wrap makes a Copy of the features passed in, overriding the UnWrap/Wrap +// method only if available in f. +func (ft *Features) Wrap(f Fs) *Features { + ftCopy := new(Features) + *ftCopy = *ft + if do, ok := f.(UnWrapper); ok { + ftCopy.UnWrap = do.UnWrap + } + if do, ok := f.(Wrapper); ok { + ftCopy.WrapFs = do.WrapFs + ftCopy.SetWrapper = do.SetWrapper + } + return ftCopy +} + +// WrapsFs adds extra information between `f` which wraps `w` +func (ft *Features) WrapsFs(f Fs, w Fs) *Features { + wFeatures := w.Features() + if wFeatures.WrapFs != nil && wFeatures.SetWrapper != nil { + wFeatures.SetWrapper(f) + } + return ft +} + +// Purger is an optional interfaces for Fs +type Purger interface { + // Purge all files in the directory specified + // + // Implement this if you have a way of deleting all the files + // quicker than just running Remove() on the result of List() + // + // Return an error if it doesn't exist + Purge(ctx context.Context, dir string) error +} + +// Copier is an optional interface for Fs +type Copier interface { + // Copy src to this remote using server-side copy operations. + // + // This is stored with the remote path given + // + // It returns the destination Object and a possible error + // + // Will only be called if src.Fs().Name() == f.Name() + // + // If it isn't possible then return fs.ErrorCantCopy + Copy(ctx context.Context, src Object, remote string) (Object, error) +} + +// Mover is an optional interface for Fs +type Mover interface { + // Move src to this remote using server-side move operations. + // + // This is stored with the remote path given + // + // It returns the destination Object and a possible error + // + // Will only be called if src.Fs().Name() == f.Name() + // + // If it isn't possible then return fs.ErrorCantMove + Move(ctx context.Context, src Object, remote string) (Object, error) +} + +// DirMover is an optional interface for Fs +type DirMover interface { + // DirMove moves src, srcRemote to this remote at dstRemote + // using server-side move operations. + // + // Will only be called if src.Fs().Name() == f.Name() + // + // If it isn't possible then return fs.ErrorCantDirMove + // + // If destination exists then return fs.ErrorDirExists + DirMove(ctx context.Context, src Fs, srcRemote, dstRemote string) error +} + +// ChangeNotifier is an optional interface for Fs +type ChangeNotifier interface { + // ChangeNotify calls the passed function with a path + // that has had changes. If the implementation + // uses polling, it should adhere to the given interval. + // At least one value will be written to the channel, + // specifying the initial value and updated values might + // follow. A 0 Duration should pause the polling. + // The ChangeNotify implementation must empty the channel + // regularly. When the channel gets closed, the implementation + // should stop polling and release resources. + ChangeNotify(context.Context, func(string, EntryType), <-chan time.Duration) +} + +// EntryType can be associated with remote paths to identify their type +type EntryType int + +// Constants +const ( + // EntryDirectory should be used to classify remote paths in directories + EntryDirectory EntryType = iota // 0 + // EntryObject should be used to classify remote paths in objects + EntryObject // 1 +) + +// UnWrapper is an optional interfaces for Fs +type UnWrapper interface { + // UnWrap returns the Fs that this Fs is wrapping + UnWrap() Fs +} + +// Wrapper is an optional interfaces for Fs +type Wrapper interface { + // Wrap returns the Fs that is wrapping this Fs + WrapFs() Fs + // SetWrapper sets the Fs that is wrapping this Fs + SetWrapper(f Fs) +} + +// DirCacheFlusher is an optional interface for Fs +type DirCacheFlusher interface { + // DirCacheFlush resets the directory cache - used in testing + // as an optional interface + DirCacheFlush() +} + +// PutUncheckeder is an optional interface for Fs +type PutUncheckeder interface { + // Put in to the remote path with the modTime given of the given size + // + // May create the object even if it returns an error - if so + // will return the object and the error, otherwise will return + // nil and the error + // + // May create duplicates or return errors if src already + // exists. + PutUnchecked(ctx context.Context, in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error) +} + +// PutStreamer is an optional interface for Fs +type PutStreamer interface { + // PutStream uploads to the remote path with the modTime given of indeterminate size + // + // May create the object even if it returns an error - if so + // will return the object and the error, otherwise will return + // nil and the error + PutStream(ctx context.Context, in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error) +} + +// PublicLinker is an optional interface for Fs +type PublicLinker interface { + // PublicLink generates a public link to the remote path (usually readable by anyone) + PublicLink(ctx context.Context, remote string, expire Duration, unlink bool) (string, error) +} + +// MergeDirser is an option interface for Fs +type MergeDirser interface { + // MergeDirs merges the contents of all the directories passed + // in into the first one and rmdirs the other directories. + MergeDirs(ctx context.Context, dirs []Directory) error +} + +// CleanUpper is an optional interfaces for Fs +type CleanUpper interface { + // CleanUp the trash in the Fs + // + // Implement this if you have a way of emptying the trash or + // otherwise cleaning up old versions of files. + CleanUp(ctx context.Context) error +} + +// ListRer is an optional interfaces for Fs +type ListRer interface { + // ListR lists the objects and directories of the Fs starting + // from dir recursively into out. + // + // dir should be "" to start from the root, and should not + // have trailing slashes. + // + // This should return ErrDirNotFound if the directory isn't + // found. + // + // It should call callback for each tranche of entries read. + // These need not be returned in any particular order. If + // callback returns an error then the listing will stop + // immediately. + // + // Don't implement this unless you have a more efficient way + // of listing recursively that doing a directory traversal. + ListR(ctx context.Context, dir string, callback ListRCallback) error +} + +// RangeSeeker is the interface that wraps the RangeSeek method. +// +// Some of the returns from Object.Open() may optionally implement +// this method for efficiency purposes. +type RangeSeeker interface { + // RangeSeek behaves like a call to Seek(offset int64, whence + // int) with the output wrapped in an io.LimitedReader + // limiting the total length to limit. + // + // RangeSeek with a limit of < 0 is equivalent to a regular Seek. + RangeSeek(ctx context.Context, offset int64, whence int, length int64) (int64, error) +} + +// Abouter is an optional interface for Fs +type Abouter interface { + // About gets quota information from the Fs + About(ctx context.Context) (*Usage, error) +} + +// OpenWriterAter is an optional interface for Fs +type OpenWriterAter interface { + // OpenWriterAt opens with a handle for random access writes + // + // Pass in the remote desired and the size if known. + // + // It truncates any existing object + OpenWriterAt(ctx context.Context, remote string, size int64) (WriterAtCloser, error) +} + +// UserInfoer is an optional interface for Fs +type UserInfoer interface { + // UserInfo returns info about the connected user + UserInfo(ctx context.Context) (map[string]string, error) +} + +// Disconnecter is an optional interface for Fs +type Disconnecter interface { + // Disconnect the current user + Disconnect(ctx context.Context) error +} + +// CommandHelp describes a single backend Command +// +// These are automatically inserted in the docs +type CommandHelp struct { + Name string // Name of the command, e.g. "link" + Short string // Single line description + Long string // Long multi-line description + Opts map[string]string // maps option name to a single line help +} + +// Commander is an interface to wrap the Command function +type Commander interface { + // Command the backend to run a named command + // + // The command run is name + // args may be used to read arguments from + // opts may be used to read optional arguments from + // + // The result should be capable of being JSON encoded + // If it is a string or a []string it will be shown to the user + // otherwise it will be JSON encoded and shown to the user like that + Command(ctx context.Context, name string, arg []string, opt map[string]string) (interface{}, error) +} + +// Shutdowner is an interface to wrap the Shutdown function +type Shutdowner interface { + // Shutdown the backend, closing any background tasks and any + // cached connections. + Shutdown(ctx context.Context) error +} + +// ObjectsChan is a channel of Objects +type ObjectsChan chan Object + +// Objects is a slice of Object~s +type Objects []Object + +// ObjectPair is a pair of Objects used to describe a potential copy +// operation. +type ObjectPair struct { + Src, Dst Object +} + +// UnWrapFs unwraps f as much as possible and returns the base Fs +func UnWrapFs(f Fs) Fs { + for { + unwrap := f.Features().UnWrap + if unwrap == nil { + break // not a wrapped Fs, use current + } + next := unwrap() + if next == nil { + break // no base Fs found, use current + } + f = next + } + return f +} + +// UnWrapObject unwraps o as much as possible and returns the base object +func UnWrapObject(o Object) Object { + for { + u, ok := o.(ObjectUnWrapper) + if !ok { + break // not a wrapped object, use current + } + next := u.UnWrap() + if next == nil { + break // no base object found, use current + } + o = next + } + return o +} + +// UnWrapObjectInfo returns the underlying Object unwrapped as much as +// possible or nil. +func UnWrapObjectInfo(oi ObjectInfo) Object { + o, ok := oi.(Object) + if !ok { + return nil + } + return UnWrapObject(o) +} diff --git a/fs/fs.go b/fs/fs.go index 5056b64e2..6ae75fd13 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -3,33 +3,13 @@ package fs import ( "context" - "crypto/md5" - "encoding/base64" - "encoding/json" - "fmt" "io" - "io/ioutil" - "log" "math" - "os" - "path/filepath" - "reflect" - "sort" - "strings" "time" "github.com/pkg/errors" - "github.com/rclone/rclone/fs/config/configmap" - "github.com/rclone/rclone/fs/config/configstruct" - "github.com/rclone/rclone/fs/fserrors" - "github.com/rclone/rclone/fs/fspath" - "github.com/rclone/rclone/fs/hash" - "github.com/rclone/rclone/lib/pacer" ) -// EntryType can be associated with remote paths to identify their type -type EntryType int - // Constants const ( // ModTimeNotSupported is a very large precision value to show @@ -37,16 +17,10 @@ const ( ModTimeNotSupported = 100 * 365 * 24 * time.Hour // MaxLevel is a sentinel representing an infinite depth for listings MaxLevel = math.MaxInt32 - // EntryDirectory should be used to classify remote paths in directories - EntryDirectory EntryType = iota // 0 - // EntryObject should be used to classify remote paths in objects - EntryObject // 1 ) // Globals var ( - // Filesystem registry - Registry []*RegInfo // ErrorNotFoundInConfigFile is returned by NewFs if not found in config file ErrorNotFoundInConfigFile = errors.New("didn't find section in config file") ErrorCantPurge = errors.New("can't purge directory") @@ -76,1410 +50,6 @@ var ( ErrorFileNameTooLong = errors.New("file name too long") ) -// RegInfo provides information about a filesystem -type RegInfo struct { - // Name of this fs - Name string - // Description of this fs - defaults to Name - Description string - // Prefix for command line flags for this fs - defaults to Name if not set - Prefix string - // Create a new file system. If root refers to an existing - // object, then it should return an Fs which which points to - // the parent of that object and ErrorIsFile. - NewFs func(ctx context.Context, name string, root string, config configmap.Mapper) (Fs, error) `json:"-"` - // Function to call to help with config - see docs for ConfigIn for more info - Config func(ctx context.Context, name string, m configmap.Mapper, configIn ConfigIn) (*ConfigOut, error) `json:"-"` - // Options for the Fs configuration - Options Options - // The command help, if any - CommandHelp []CommandHelp -} - -// FileName returns the on disk file name for this backend -func (ri *RegInfo) FileName() string { - return strings.Replace(ri.Name, " ", "", -1) -} - -// Options is a slice of configuration Option for a backend -type Options []Option - -// Set the default values for the options -func (os Options) setValues() { - for i := range os { - o := &os[i] - if o.Default == nil { - o.Default = "" - } - } -} - -// Get the Option corresponding to name or return nil if not found -func (os Options) Get(name string) *Option { - for i := range os { - opt := &os[i] - if opt.Name == name { - return opt - } - } - return nil -} - -// Overridden discovers which config items have been overridden in the -// configmap passed in, either by the config string, command line -// flags or environment variables -func (os Options) Overridden(m *configmap.Map) configmap.Simple { - var overridden = configmap.Simple{} - for i := range os { - opt := &os[i] - value, isSet := m.GetPriority(opt.Name, configmap.PriorityNormal) - if isSet { - overridden.Set(opt.Name, value) - } - } - return overridden -} - -// NonDefault discovers which config values aren't at their default -func (os Options) NonDefault(m configmap.Getter) configmap.Simple { - var nonDefault = configmap.Simple{} - for i := range os { - opt := &os[i] - value, isSet := m.Get(opt.Name) - if !isSet { - continue - } - defaultValue := fmt.Sprint(opt.Default) - if value != defaultValue { - nonDefault.Set(opt.Name, value) - } - } - return nonDefault -} - -// HasAdvanced discovers if any options have an Advanced setting -func (os Options) HasAdvanced() bool { - for i := range os { - opt := &os[i] - if opt.Advanced { - return true - } - } - return false -} - -// OptionVisibility controls whether the options are visible in the -// configurator or the command line. -type OptionVisibility byte - -// Constants Option.Hide -const ( - OptionHideCommandLine OptionVisibility = 1 << iota - OptionHideConfigurator - OptionHideBoth = OptionHideCommandLine | OptionHideConfigurator -) - -// Option is describes an option for the config wizard -// -// This also describes command line options and environment variables -type Option struct { - Name string // name of the option in snake_case - Help string // Help, the first line only is used for the command line help - Provider string // Set to filter on provider - Default interface{} // default value, nil => "" - Value interface{} // value to be set by flags - Examples OptionExamples `json:",omitempty"` // config examples - ShortOpt string // the short option for this if required - Hide OptionVisibility // set this to hide the config from the configurator or the command line - Required bool // this option is required - IsPassword bool // set if the option is a password - NoPrefix bool // set if the option for this should not use the backend prefix - Advanced bool // set if this is an advanced config option - Exclusive bool // set if the answer can only be one of the examples -} - -// BaseOption is an alias for Option used internally -type BaseOption Option - -// MarshalJSON turns an Option into JSON -// -// It adds some generated fields for ease of use -// - DefaultStr - a string rendering of Default -// - ValueStr - a string rendering of Value -// - Type - the type of the option -func (o *Option) MarshalJSON() ([]byte, error) { - return json.Marshal(struct { - BaseOption - DefaultStr string - ValueStr string - Type string - }{ - BaseOption: BaseOption(*o), - DefaultStr: fmt.Sprint(o.Default), - ValueStr: o.String(), - Type: o.Type(), - }) -} - -// GetValue gets the current current value which is the default if not set -func (o *Option) GetValue() interface{} { - val := o.Value - if val == nil { - val = o.Default - if val == nil { - val = "" - } - } - return val -} - -// String turns Option into a string -func (o *Option) String() string { - return fmt.Sprint(o.GetValue()) -} - -// Set an Option from a string -func (o *Option) Set(s string) (err error) { - newValue, err := configstruct.StringToInterface(o.GetValue(), s) - if err != nil { - return err - } - o.Value = newValue - return nil -} - -// Type of the value -func (o *Option) Type() string { - return reflect.TypeOf(o.GetValue()).Name() -} - -// FlagName for the option -func (o *Option) FlagName(prefix string) string { - name := strings.Replace(o.Name, "_", "-", -1) // convert snake_case to kebab-case - if !o.NoPrefix { - name = prefix + "-" + name - } - return name -} - -// EnvVarName for the option -func (o *Option) EnvVarName(prefix string) string { - return OptionToEnv(prefix + "-" + o.Name) -} - -// Copy makes a shallow copy of the option -func (o *Option) Copy() *Option { - copy := new(Option) - *copy = *o - return copy -} - -// OptionExamples is a slice of examples -type OptionExamples []OptionExample - -// Len is part of sort.Interface. -func (os OptionExamples) Len() int { return len(os) } - -// Swap is part of sort.Interface. -func (os OptionExamples) Swap(i, j int) { os[i], os[j] = os[j], os[i] } - -// Less is part of sort.Interface. -func (os OptionExamples) Less(i, j int) bool { return os[i].Help < os[j].Help } - -// Sort sorts an OptionExamples -func (os OptionExamples) Sort() { sort.Sort(os) } - -// OptionExample describes an example for an Option -type OptionExample struct { - Value string - Help string - Provider string -} - -// Register a filesystem -// -// Fs modules should use this in an init() function -func Register(info *RegInfo) { - info.Options.setValues() - if info.Prefix == "" { - info.Prefix = info.Name - } - Registry = append(Registry, info) -} - -// Fs is the interface a cloud storage system must provide -type Fs interface { - Info - - // List the objects and directories in dir into entries. The - // entries can be returned in any order but should be for a - // complete directory. - // - // dir should be "" to list the root, and should not have - // trailing slashes. - // - // This should return ErrDirNotFound if the directory isn't - // found. - List(ctx context.Context, dir string) (entries DirEntries, err error) - - // NewObject finds the Object at remote. If it can't be found - // it returns the error ErrorObjectNotFound. - NewObject(ctx context.Context, remote string) (Object, error) - - // Put in to the remote path with the modTime given of the given size - // - // When called from outside an Fs by rclone, src.Size() will always be >= 0. - // But for unknown-sized objects (indicated by src.Size() == -1), Put should either - // return an error or upload it properly (rather than e.g. calling panic). - // - // May create the object even if it returns an error - if so - // will return the object and the error, otherwise will return - // nil and the error - Put(ctx context.Context, in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error) - - // Mkdir makes the directory (container, bucket) - // - // Shouldn't return an error if it already exists - Mkdir(ctx context.Context, dir string) error - - // Rmdir removes the directory (container, bucket) if empty - // - // Return an error if it doesn't exist or isn't empty - Rmdir(ctx context.Context, dir string) error -} - -// Info provides a read only interface to information about a filesystem. -type Info interface { - // Name of the remote (as passed into NewFs) - Name() string - - // Root of the remote (as passed into NewFs) - Root() string - - // String returns a description of the FS - String() string - - // Precision of the ModTimes in this Fs - Precision() time.Duration - - // Returns the supported hash types of the filesystem - Hashes() hash.Set - - // Features returns the optional features of this Fs - Features() *Features -} - -// Object is a filesystem like object provided by an Fs -type Object interface { - ObjectInfo - - // SetModTime sets the metadata on the object to set the modification date - SetModTime(ctx context.Context, t time.Time) error - - // Open opens the file for read. Call Close() on the returned io.ReadCloser - Open(ctx context.Context, options ...OpenOption) (io.ReadCloser, error) - - // Update in to the object with the modTime given of the given size - // - // When called from outside an Fs by rclone, src.Size() will always be >= 0. - // But for unknown-sized objects (indicated by src.Size() == -1), Upload should either - // return an error or update the object properly (rather than e.g. calling panic). - Update(ctx context.Context, in io.Reader, src ObjectInfo, options ...OpenOption) error - - // Removes this object - Remove(ctx context.Context) error -} - -// ObjectInfo provides read only information about an object. -type ObjectInfo interface { - DirEntry - - // Fs returns read only access to the Fs that this object is part of - Fs() Info - - // Hash returns the selected checksum of the file - // If no checksum is available it returns "" - Hash(ctx context.Context, ty hash.Type) (string, error) - - // Storable says whether this object can be stored - Storable() bool -} - -// DirEntry provides read only information about the common subset of -// a Dir or Object. These are returned from directory listings - type -// assert them into the correct type. -type DirEntry interface { - // String returns a description of the Object - String() string - - // Remote returns the remote path - Remote() string - - // ModTime returns the modification date of the file - // It should return a best guess if one isn't available - ModTime(context.Context) time.Time - - // Size returns the size of the file - Size() int64 -} - -// Directory is a filesystem like directory provided by an Fs -type Directory interface { - DirEntry - - // Items returns the count of items in this directory or this - // directory and subdirectories if known, -1 for unknown - Items() int64 - - // ID returns the internal ID of this directory if known, or - // "" otherwise - ID() string -} - -// MimeTyper is an optional interface for Object -type MimeTyper interface { - // MimeType returns the content type of the Object if - // known, or "" if not - MimeType(ctx context.Context) string -} - -// IDer is an optional interface for Object -type IDer interface { - // ID returns the ID of the Object if known, or "" if not - ID() string -} - -// ParentIDer is an optional interface for Object -type ParentIDer interface { - // ParentID returns the ID of the parent directory if known or nil if not - ParentID() string -} - -// ObjectUnWrapper is an optional interface for Object -type ObjectUnWrapper interface { - // UnWrap returns the Object that this Object is wrapping or - // nil if it isn't wrapping anything - UnWrap() Object -} - -// SetTierer is an optional interface for Object -type SetTierer interface { - // SetTier performs changing storage tier of the Object if - // multiple storage classes supported - SetTier(tier string) error -} - -// GetTierer is an optional interface for Object -type GetTierer interface { - // GetTier returns storage tier or class of the Object - GetTier() string -} - -// FullObjectInfo contains all the read-only optional interfaces -// -// Use for checking making wrapping ObjectInfos implement everything -type FullObjectInfo interface { - ObjectInfo - MimeTyper - IDer - ObjectUnWrapper - GetTierer -} - -// FullObject contains all the optional interfaces for Object -// -// Use for checking making wrapping Objects implement everything -type FullObject interface { - Object - MimeTyper - IDer - ObjectUnWrapper - GetTierer - SetTierer -} - -// ObjectOptionalInterfaces returns the names of supported and -// unsupported optional interfaces for an Object -func ObjectOptionalInterfaces(o Object) (supported, unsupported []string) { - store := func(ok bool, name string) { - if ok { - supported = append(supported, name) - } else { - unsupported = append(unsupported, name) - } - } - - _, ok := o.(MimeTyper) - store(ok, "MimeType") - - _, ok = o.(IDer) - store(ok, "ID") - - _, ok = o.(ObjectUnWrapper) - store(ok, "UnWrap") - - _, ok = o.(SetTierer) - store(ok, "SetTier") - - _, ok = o.(GetTierer) - store(ok, "GetTier") - - return supported, unsupported -} - -// ListRCallback defines a callback function for ListR to use -// -// It is called for each tranche of entries read from the listing and -// if it returns an error, the listing stops. -type ListRCallback func(entries DirEntries) error - -// ListRFn is defines the call used to recursively list a directory -type ListRFn func(ctx context.Context, dir string, callback ListRCallback) error - -// NewUsageValue makes a valid value -func NewUsageValue(value int64) *int64 { - p := new(int64) - *p = value - return p -} - -// Usage is returned by the About call -// -// If a value is nil then it isn't supported by that backend -type Usage struct { - Total *int64 `json:"total,omitempty"` // quota of bytes that can be used - Used *int64 `json:"used,omitempty"` // bytes in use - Trashed *int64 `json:"trashed,omitempty"` // bytes in trash - Other *int64 `json:"other,omitempty"` // other usage e.g. gmail in drive - Free *int64 `json:"free,omitempty"` // bytes which can be uploaded before reaching the quota - Objects *int64 `json:"objects,omitempty"` // objects in the storage system -} - -// WriterAtCloser wraps io.WriterAt and io.Closer -type WriterAtCloser interface { - io.WriterAt - io.Closer -} - -// Features describe the optional features of the Fs -type Features struct { - // Feature flags, whether Fs - CaseInsensitive bool // has case insensitive files - DuplicateFiles bool // allows duplicate files - ReadMimeType bool // can read the mime type of objects - WriteMimeType bool // can set the mime type of objects - CanHaveEmptyDirectories bool // can have empty directories - BucketBased bool // is bucket based (like s3, swift, etc.) - BucketBasedRootOK bool // is bucket based and can use from root - SetTier bool // allows set tier functionality on objects - GetTier bool // allows to retrieve storage tier of objects - ServerSideAcrossConfigs bool // can server-side copy between different remotes of the same type - IsLocal bool // is the local backend - SlowModTime bool // if calling ModTime() generally takes an extra transaction - SlowHash bool // if calling Hash() generally takes an extra transaction - - // Purge all files in the directory specified - // - // Implement this if you have a way of deleting all the files - // quicker than just running Remove() on the result of List() - // - // Return an error if it doesn't exist - Purge func(ctx context.Context, dir string) error - - // Copy src to this remote using server-side copy operations. - // - // This is stored with the remote path given - // - // It returns the destination Object and a possible error - // - // Will only be called if src.Fs().Name() == f.Name() - // - // If it isn't possible then return fs.ErrorCantCopy - Copy func(ctx context.Context, src Object, remote string) (Object, error) - - // Move src to this remote using server-side move operations. - // - // This is stored with the remote path given - // - // It returns the destination Object and a possible error - // - // Will only be called if src.Fs().Name() == f.Name() - // - // If it isn't possible then return fs.ErrorCantMove - Move func(ctx context.Context, src Object, remote string) (Object, error) - - // DirMove moves src, srcRemote to this remote at dstRemote - // using server-side move operations. - // - // Will only be called if src.Fs().Name() == f.Name() - // - // If it isn't possible then return fs.ErrorCantDirMove - // - // If destination exists then return fs.ErrorDirExists - DirMove func(ctx context.Context, src Fs, srcRemote, dstRemote string) error - - // ChangeNotify calls the passed function with a path - // that has had changes. If the implementation - // uses polling, it should adhere to the given interval. - ChangeNotify func(context.Context, func(string, EntryType), <-chan time.Duration) - - // UnWrap returns the Fs that this Fs is wrapping - UnWrap func() Fs - - // WrapFs returns the Fs that is wrapping this Fs - WrapFs func() Fs - - // SetWrapper sets the Fs that is wrapping this Fs - SetWrapper func(f Fs) - - // DirCacheFlush resets the directory cache - used in testing - // as an optional interface - DirCacheFlush func() - - // PublicLink generates a public link to the remote path (usually readable by anyone) - PublicLink func(ctx context.Context, remote string, expire Duration, unlink bool) (string, error) - - // Put in to the remote path with the modTime given of the given size - // - // May create the object even if it returns an error - if so - // will return the object and the error, otherwise will return - // nil and the error - // - // May create duplicates or return errors if src already - // exists. - PutUnchecked func(ctx context.Context, in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error) - - // PutStream uploads to the remote path with the modTime given of indeterminate size - // - // May create the object even if it returns an error - if so - // will return the object and the error, otherwise will return - // nil and the error - PutStream func(ctx context.Context, in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error) - - // MergeDirs merges the contents of all the directories passed - // in into the first one and rmdirs the other directories. - MergeDirs func(ctx context.Context, dirs []Directory) error - - // CleanUp the trash in the Fs - // - // Implement this if you have a way of emptying the trash or - // otherwise cleaning up old versions of files. - CleanUp func(ctx context.Context) error - - // ListR lists the objects and directories of the Fs starting - // from dir recursively into out. - // - // dir should be "" to start from the root, and should not - // have trailing slashes. - // - // This should return ErrDirNotFound if the directory isn't - // found. - // - // It should call callback for each tranche of entries read. - // These need not be returned in any particular order. If - // callback returns an error then the listing will stop - // immediately. - // - // Don't implement this unless you have a more efficient way - // of listing recursively that doing a directory traversal. - ListR ListRFn - - // About gets quota information from the Fs - About func(ctx context.Context) (*Usage, error) - - // OpenWriterAt opens with a handle for random access writes - // - // Pass in the remote desired and the size if known. - // - // It truncates any existing object - OpenWriterAt func(ctx context.Context, remote string, size int64) (WriterAtCloser, error) - - // UserInfo returns info about the connected user - UserInfo func(ctx context.Context) (map[string]string, error) - - // Disconnect the current user - Disconnect func(ctx context.Context) error - - // Command the backend to run a named command - // - // The command run is name - // args may be used to read arguments from - // opts may be used to read optional arguments from - // - // The result should be capable of being JSON encoded - // If it is a string or a []string it will be shown to the user - // otherwise it will be JSON encoded and shown to the user like that - Command func(ctx context.Context, name string, arg []string, opt map[string]string) (interface{}, error) - - // Shutdown the backend, closing any background tasks and any - // cached connections. - Shutdown func(ctx context.Context) error -} - -// Disable nil's out the named feature. If it isn't found then it -// will log a message. -func (ft *Features) Disable(name string) *Features { - v := reflect.ValueOf(ft).Elem() - vType := v.Type() - for i := 0; i < v.NumField(); i++ { - vName := vType.Field(i).Name - field := v.Field(i) - if strings.EqualFold(name, vName) { - if !field.CanSet() { - Errorf(nil, "Can't set Feature %q", name) - } else { - zero := reflect.Zero(field.Type()) - field.Set(zero) - Debugf(nil, "Reset feature %q", name) - } - } - } - return ft -} - -// List returns a slice of all the possible feature names -func (ft *Features) List() (out []string) { - v := reflect.ValueOf(ft).Elem() - vType := v.Type() - for i := 0; i < v.NumField(); i++ { - out = append(out, vType.Field(i).Name) - } - return out -} - -// Enabled returns a map of features with keys showing whether they -// are enabled or not -func (ft *Features) Enabled() (features map[string]bool) { - v := reflect.ValueOf(ft).Elem() - vType := v.Type() - features = make(map[string]bool, v.NumField()) - for i := 0; i < v.NumField(); i++ { - vName := vType.Field(i).Name - field := v.Field(i) - if field.Kind() == reflect.Func { - // Can't compare functions - features[vName] = !field.IsNil() - } else { - zero := reflect.Zero(field.Type()) - features[vName] = field.Interface() != zero.Interface() - } - } - return features -} - -// DisableList nil's out the comma separated list of named features. -// If it isn't found then it will log a message. -func (ft *Features) DisableList(list []string) *Features { - for _, feature := range list { - ft.Disable(strings.TrimSpace(feature)) - } - return ft -} - -// Fill fills in the function pointers in the Features struct from the -// optional interfaces. It returns the original updated Features -// struct passed in. -func (ft *Features) Fill(ctx context.Context, f Fs) *Features { - if do, ok := f.(Purger); ok { - ft.Purge = do.Purge - } - if do, ok := f.(Copier); ok { - ft.Copy = do.Copy - } - if do, ok := f.(Mover); ok { - ft.Move = do.Move - } - if do, ok := f.(DirMover); ok { - ft.DirMove = do.DirMove - } - if do, ok := f.(ChangeNotifier); ok { - ft.ChangeNotify = do.ChangeNotify - } - if do, ok := f.(UnWrapper); ok { - ft.UnWrap = do.UnWrap - } - if do, ok := f.(Wrapper); ok { - ft.WrapFs = do.WrapFs - ft.SetWrapper = do.SetWrapper - } - if do, ok := f.(DirCacheFlusher); ok { - ft.DirCacheFlush = do.DirCacheFlush - } - if do, ok := f.(PublicLinker); ok { - ft.PublicLink = do.PublicLink - } - if do, ok := f.(PutUncheckeder); ok { - ft.PutUnchecked = do.PutUnchecked - } - if do, ok := f.(PutStreamer); ok { - ft.PutStream = do.PutStream - } - if do, ok := f.(MergeDirser); ok { - ft.MergeDirs = do.MergeDirs - } - if do, ok := f.(CleanUpper); ok { - ft.CleanUp = do.CleanUp - } - if do, ok := f.(ListRer); ok { - ft.ListR = do.ListR - } - if do, ok := f.(Abouter); ok { - ft.About = do.About - } - if do, ok := f.(OpenWriterAter); ok { - ft.OpenWriterAt = do.OpenWriterAt - } - if do, ok := f.(UserInfoer); ok { - ft.UserInfo = do.UserInfo - } - if do, ok := f.(Disconnecter); ok { - ft.Disconnect = do.Disconnect - } - if do, ok := f.(Commander); ok { - ft.Command = do.Command - } - if do, ok := f.(Shutdowner); ok { - ft.Shutdown = do.Shutdown - } - return ft.DisableList(GetConfig(ctx).DisableFeatures) -} - -// Mask the Features with the Fs passed in -// -// Only optional features which are implemented in both the original -// Fs AND the one passed in will be advertised. Any features which -// aren't in both will be set to false/nil, except for UnWrap/Wrap which -// will be left untouched. -func (ft *Features) Mask(ctx context.Context, f Fs) *Features { - mask := f.Features() - ft.CaseInsensitive = ft.CaseInsensitive && mask.CaseInsensitive - ft.DuplicateFiles = ft.DuplicateFiles && mask.DuplicateFiles - ft.ReadMimeType = ft.ReadMimeType && mask.ReadMimeType - ft.WriteMimeType = ft.WriteMimeType && mask.WriteMimeType - ft.CanHaveEmptyDirectories = ft.CanHaveEmptyDirectories && mask.CanHaveEmptyDirectories - ft.BucketBased = ft.BucketBased && mask.BucketBased - ft.BucketBasedRootOK = ft.BucketBasedRootOK && mask.BucketBasedRootOK - ft.SetTier = ft.SetTier && mask.SetTier - ft.GetTier = ft.GetTier && mask.GetTier - ft.ServerSideAcrossConfigs = ft.ServerSideAcrossConfigs && mask.ServerSideAcrossConfigs - // ft.IsLocal = ft.IsLocal && mask.IsLocal Don't propagate IsLocal - ft.SlowModTime = ft.SlowModTime && mask.SlowModTime - ft.SlowHash = ft.SlowHash && mask.SlowHash - - if mask.Purge == nil { - ft.Purge = nil - } - if mask.Copy == nil { - ft.Copy = nil - } - if mask.Move == nil { - ft.Move = nil - } - if mask.DirMove == nil { - ft.DirMove = nil - } - if mask.ChangeNotify == nil { - ft.ChangeNotify = nil - } - // if mask.UnWrap == nil { - // ft.UnWrap = nil - // } - // if mask.Wrapper == nil { - // ft.Wrapper = nil - // } - if mask.DirCacheFlush == nil { - ft.DirCacheFlush = nil - } - if mask.PublicLink == nil { - ft.PublicLink = nil - } - if mask.PutUnchecked == nil { - ft.PutUnchecked = nil - } - if mask.PutStream == nil { - ft.PutStream = nil - } - if mask.MergeDirs == nil { - ft.MergeDirs = nil - } - if mask.CleanUp == nil { - ft.CleanUp = nil - } - if mask.ListR == nil { - ft.ListR = nil - } - if mask.About == nil { - ft.About = nil - } - if mask.OpenWriterAt == nil { - ft.OpenWriterAt = nil - } - if mask.UserInfo == nil { - ft.UserInfo = nil - } - if mask.Disconnect == nil { - ft.Disconnect = nil - } - // Command is always local so we don't mask it - if mask.Shutdown == nil { - ft.Shutdown = nil - } - return ft.DisableList(GetConfig(ctx).DisableFeatures) -} - -// Wrap makes a Copy of the features passed in, overriding the UnWrap/Wrap -// method only if available in f. -func (ft *Features) Wrap(f Fs) *Features { - ftCopy := new(Features) - *ftCopy = *ft - if do, ok := f.(UnWrapper); ok { - ftCopy.UnWrap = do.UnWrap - } - if do, ok := f.(Wrapper); ok { - ftCopy.WrapFs = do.WrapFs - ftCopy.SetWrapper = do.SetWrapper - } - return ftCopy -} - -// WrapsFs adds extra information between `f` which wraps `w` -func (ft *Features) WrapsFs(f Fs, w Fs) *Features { - wFeatures := w.Features() - if wFeatures.WrapFs != nil && wFeatures.SetWrapper != nil { - wFeatures.SetWrapper(f) - } - return ft -} - -// Purger is an optional interfaces for Fs -type Purger interface { - // Purge all files in the directory specified - // - // Implement this if you have a way of deleting all the files - // quicker than just running Remove() on the result of List() - // - // Return an error if it doesn't exist - Purge(ctx context.Context, dir string) error -} - -// Copier is an optional interface for Fs -type Copier interface { - // Copy src to this remote using server-side copy operations. - // - // This is stored with the remote path given - // - // It returns the destination Object and a possible error - // - // Will only be called if src.Fs().Name() == f.Name() - // - // If it isn't possible then return fs.ErrorCantCopy - Copy(ctx context.Context, src Object, remote string) (Object, error) -} - -// Mover is an optional interface for Fs -type Mover interface { - // Move src to this remote using server-side move operations. - // - // This is stored with the remote path given - // - // It returns the destination Object and a possible error - // - // Will only be called if src.Fs().Name() == f.Name() - // - // If it isn't possible then return fs.ErrorCantMove - Move(ctx context.Context, src Object, remote string) (Object, error) -} - -// DirMover is an optional interface for Fs -type DirMover interface { - // DirMove moves src, srcRemote to this remote at dstRemote - // using server-side move operations. - // - // Will only be called if src.Fs().Name() == f.Name() - // - // If it isn't possible then return fs.ErrorCantDirMove - // - // If destination exists then return fs.ErrorDirExists - DirMove(ctx context.Context, src Fs, srcRemote, dstRemote string) error -} - -// ChangeNotifier is an optional interface for Fs -type ChangeNotifier interface { - // ChangeNotify calls the passed function with a path - // that has had changes. If the implementation - // uses polling, it should adhere to the given interval. - // At least one value will be written to the channel, - // specifying the initial value and updated values might - // follow. A 0 Duration should pause the polling. - // The ChangeNotify implementation must empty the channel - // regularly. When the channel gets closed, the implementation - // should stop polling and release resources. - ChangeNotify(context.Context, func(string, EntryType), <-chan time.Duration) -} - -// UnWrapper is an optional interfaces for Fs -type UnWrapper interface { - // UnWrap returns the Fs that this Fs is wrapping - UnWrap() Fs -} - -// Wrapper is an optional interfaces for Fs -type Wrapper interface { - // Wrap returns the Fs that is wrapping this Fs - WrapFs() Fs - // SetWrapper sets the Fs that is wrapping this Fs - SetWrapper(f Fs) -} - -// DirCacheFlusher is an optional interface for Fs -type DirCacheFlusher interface { - // DirCacheFlush resets the directory cache - used in testing - // as an optional interface - DirCacheFlush() -} - -// PutUncheckeder is an optional interface for Fs -type PutUncheckeder interface { - // Put in to the remote path with the modTime given of the given size - // - // May create the object even if it returns an error - if so - // will return the object and the error, otherwise will return - // nil and the error - // - // May create duplicates or return errors if src already - // exists. - PutUnchecked(ctx context.Context, in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error) -} - -// PutStreamer is an optional interface for Fs -type PutStreamer interface { - // PutStream uploads to the remote path with the modTime given of indeterminate size - // - // May create the object even if it returns an error - if so - // will return the object and the error, otherwise will return - // nil and the error - PutStream(ctx context.Context, in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error) -} - -// PublicLinker is an optional interface for Fs -type PublicLinker interface { - // PublicLink generates a public link to the remote path (usually readable by anyone) - PublicLink(ctx context.Context, remote string, expire Duration, unlink bool) (string, error) -} - -// MergeDirser is an option interface for Fs -type MergeDirser interface { - // MergeDirs merges the contents of all the directories passed - // in into the first one and rmdirs the other directories. - MergeDirs(ctx context.Context, dirs []Directory) error -} - -// CleanUpper is an optional interfaces for Fs -type CleanUpper interface { - // CleanUp the trash in the Fs - // - // Implement this if you have a way of emptying the trash or - // otherwise cleaning up old versions of files. - CleanUp(ctx context.Context) error -} - -// ListRer is an optional interfaces for Fs -type ListRer interface { - // ListR lists the objects and directories of the Fs starting - // from dir recursively into out. - // - // dir should be "" to start from the root, and should not - // have trailing slashes. - // - // This should return ErrDirNotFound if the directory isn't - // found. - // - // It should call callback for each tranche of entries read. - // These need not be returned in any particular order. If - // callback returns an error then the listing will stop - // immediately. - // - // Don't implement this unless you have a more efficient way - // of listing recursively that doing a directory traversal. - ListR(ctx context.Context, dir string, callback ListRCallback) error -} - -// RangeSeeker is the interface that wraps the RangeSeek method. -// -// Some of the returns from Object.Open() may optionally implement -// this method for efficiency purposes. -type RangeSeeker interface { - // RangeSeek behaves like a call to Seek(offset int64, whence - // int) with the output wrapped in an io.LimitedReader - // limiting the total length to limit. - // - // RangeSeek with a limit of < 0 is equivalent to a regular Seek. - RangeSeek(ctx context.Context, offset int64, whence int, length int64) (int64, error) -} - -// Abouter is an optional interface for Fs -type Abouter interface { - // About gets quota information from the Fs - About(ctx context.Context) (*Usage, error) -} - -// OpenWriterAter is an optional interface for Fs -type OpenWriterAter interface { - // OpenWriterAt opens with a handle for random access writes - // - // Pass in the remote desired and the size if known. - // - // It truncates any existing object - OpenWriterAt(ctx context.Context, remote string, size int64) (WriterAtCloser, error) -} - -// UserInfoer is an optional interface for Fs -type UserInfoer interface { - // UserInfo returns info about the connected user - UserInfo(ctx context.Context) (map[string]string, error) -} - -// Disconnecter is an optional interface for Fs -type Disconnecter interface { - // Disconnect the current user - Disconnect(ctx context.Context) error -} - -// CommandHelp describes a single backend Command -// -// These are automatically inserted in the docs -type CommandHelp struct { - Name string // Name of the command, e.g. "link" - Short string // Single line description - Long string // Long multi-line description - Opts map[string]string // maps option name to a single line help -} - -// Commander is an interface to wrap the Command function -type Commander interface { - // Command the backend to run a named command - // - // The command run is name - // args may be used to read arguments from - // opts may be used to read optional arguments from - // - // The result should be capable of being JSON encoded - // If it is a string or a []string it will be shown to the user - // otherwise it will be JSON encoded and shown to the user like that - Command(ctx context.Context, name string, arg []string, opt map[string]string) (interface{}, error) -} - -// Shutdowner is an interface to wrap the Shutdown function -type Shutdowner interface { - // Shutdown the backend, closing any background tasks and any - // cached connections. - Shutdown(ctx context.Context) error -} - -// ObjectsChan is a channel of Objects -type ObjectsChan chan Object - -// Objects is a slice of Object~s -type Objects []Object - -// ObjectPair is a pair of Objects used to describe a potential copy -// operation. -type ObjectPair struct { - Src, Dst Object -} - -// UnWrapFs unwraps f as much as possible and returns the base Fs -func UnWrapFs(f Fs) Fs { - for { - unwrap := f.Features().UnWrap - if unwrap == nil { - break // not a wrapped Fs, use current - } - next := unwrap() - if next == nil { - break // no base Fs found, use current - } - f = next - } - return f -} - -// UnWrapObject unwraps o as much as possible and returns the base object -func UnWrapObject(o Object) Object { - for { - u, ok := o.(ObjectUnWrapper) - if !ok { - break // not a wrapped object, use current - } - next := u.UnWrap() - if next == nil { - break // no base object found, use current - } - o = next - } - return o -} - -// UnWrapObjectInfo returns the underlying Object unwrapped as much as -// possible or nil. -func UnWrapObjectInfo(oi ObjectInfo) Object { - o, ok := oi.(Object) - if !ok { - return nil - } - return UnWrapObject(o) -} - -// Find looks for a RegInfo object for the name passed in. The name -// can be either the Name or the Prefix. -// -// Services are looked up in the config file -func Find(name string) (*RegInfo, error) { - for _, item := range Registry { - if item.Name == name || item.Prefix == name || item.FileName() == name { - return item, nil - } - } - return nil, errors.Errorf("didn't find backend called %q", name) -} - -// MustFind looks for an Info object for the type name passed in -// -// Services are looked up in the config file -// -// Exits with a fatal error if not found -func MustFind(name string) *RegInfo { - fs, err := Find(name) - if err != nil { - log.Fatalf("Failed to find remote: %v", err) - } - return fs -} - -// ParseRemote deconstructs a path into configName, fsPath, looking up -// the fsName in the config file (returning NotFoundInConfigFile if not found) -func ParseRemote(path string) (fsInfo *RegInfo, configName, fsPath string, connectionStringConfig configmap.Simple, err error) { - parsed, err := fspath.Parse(path) - if err != nil { - return nil, "", "", nil, err - } - configName, fsPath = parsed.Name, parsed.Path - var fsName string - var ok bool - if configName != "" { - if strings.HasPrefix(configName, ":") { - fsName = configName[1:] - } else { - m := ConfigMap(nil, configName, parsed.Config) - fsName, ok = m.Get("type") - if !ok { - return nil, "", "", nil, ErrorNotFoundInConfigFile - } - } - } else { - fsName = "local" - configName = "local" - } - fsInfo, err = Find(fsName) - return fsInfo, configName, fsPath, parsed.Config, err -} - -// A configmap.Getter to read from the environment RCLONE_CONFIG_backend_option_name -type configEnvVars string - -// Get a config item from the environment variables if possible -func (configName configEnvVars) Get(key string) (value string, ok bool) { - return os.LookupEnv(ConfigToEnv(string(configName), key)) -} - -// A configmap.Getter to read from the environment RCLONE_option_name -type optionEnvVars struct { - fsInfo *RegInfo -} - -// Get a config item from the option environment variables if possible -func (oev optionEnvVars) Get(key string) (value string, ok bool) { - opt := oev.fsInfo.Options.Get(key) - if opt == nil { - return "", false - } - // For options with NoPrefix set, check without prefix too - if opt.NoPrefix { - value, ok = os.LookupEnv(OptionToEnv(key)) - if ok { - return value, ok - } - } - return os.LookupEnv(OptionToEnv(oev.fsInfo.Prefix + "-" + key)) -} - -// A configmap.Getter to read either the default value or the set -// value from the RegInfo.Options -type regInfoValues struct { - fsInfo *RegInfo - useDefault bool -} - -// override the values in configMap with the either the flag values or -// the default values -func (r *regInfoValues) Get(key string) (value string, ok bool) { - opt := r.fsInfo.Options.Get(key) - if opt != nil && (r.useDefault || opt.Value != nil) { - return opt.String(), true - } - return "", false -} - -// A configmap.Setter to read from the config file -type setConfigFile string - -// Set a config item into the config file -func (section setConfigFile) Set(key, value string) { - if strings.HasPrefix(string(section), ":") { - Logf(nil, "Can't save config %q = %q for on the fly backend %q", key, value, section) - return - } - Debugf(nil, "Saving config %q = %q in section %q of the config file", key, value, section) - err := ConfigFileSet(string(section), key, value) - if err != nil { - Errorf(nil, "Failed saving config %q = %q in section %q of the config file: %v", key, value, section, err) - } -} - -// A configmap.Getter to read from the config file -type getConfigFile string - -// Get a config item from the config file -func (section getConfigFile) Get(key string) (value string, ok bool) { - value, ok = ConfigFileGet(string(section), key) - // Ignore empty lines in the config file - if value == "" { - ok = false - } - return value, ok -} - -// ConfigMap creates a configmap.Map from the *RegInfo and the -// configName passed in. If connectionStringConfig has any entries (it may be nil), -// then it will be added to the lookup with the highest priority. -// -// If fsInfo is nil then the returned configmap.Map should only be -// used for reading non backend specific parameters, such as "type". -func ConfigMap(fsInfo *RegInfo, configName string, connectionStringConfig configmap.Simple) (config *configmap.Map) { - // Create the config - config = configmap.New() - - // Read the config, more specific to least specific - - // Config from connection string - if len(connectionStringConfig) > 0 { - config.AddGetter(connectionStringConfig, configmap.PriorityNormal) - } - - // flag values - if fsInfo != nil { - config.AddGetter(®InfoValues{fsInfo, false}, configmap.PriorityNormal) - } - - // remote specific environment vars - config.AddGetter(configEnvVars(configName), configmap.PriorityNormal) - - // backend specific environment vars - if fsInfo != nil { - config.AddGetter(optionEnvVars{fsInfo: fsInfo}, configmap.PriorityNormal) - } - - // config file - config.AddGetter(getConfigFile(configName), configmap.PriorityConfig) - - // default values - if fsInfo != nil { - config.AddGetter(®InfoValues{fsInfo, true}, configmap.PriorityDefault) - } - - // Set Config - config.AddSetter(setConfigFile(configName)) - return config -} - -// ConfigFs makes the config for calling NewFs with. -// -// It parses the path which is of the form remote:path -// -// Remotes are looked up in the config file. If the remote isn't -// found then NotFoundInConfigFile will be returned. -func ConfigFs(path string) (fsInfo *RegInfo, configName, fsPath string, config *configmap.Map, err error) { - // Parse the remote path - fsInfo, configName, fsPath, connectionStringConfig, err := ParseRemote(path) - if err != nil { - return - } - config = ConfigMap(fsInfo, configName, connectionStringConfig) - return -} - -// NewFs makes a new Fs object from the path -// -// The path is of the form remote:path -// -// Remotes are looked up in the config file. If the remote isn't -// found then NotFoundInConfigFile will be returned. -// -// On Windows avoid single character remote names as they can be mixed -// up with drive letters. -func NewFs(ctx context.Context, path string) (Fs, error) { - Debugf(nil, "Creating backend with remote %q", path) - fsInfo, configName, fsPath, config, err := ConfigFs(path) - if err != nil { - return nil, err - } - overridden := fsInfo.Options.Overridden(config) - if len(overridden) > 0 { - extraConfig := overridden.String() - //Debugf(nil, "detected overriden config %q", extraConfig) - md5sumBinary := md5.Sum([]byte(extraConfig)) - suffix := base64.RawURLEncoding.EncodeToString(md5sumBinary[:]) - // 5 characters length is 5*6 = 30 bits of base64 - const maxLength = 5 - if len(suffix) > maxLength { - suffix = suffix[:maxLength] - } - suffix = "{" + suffix + "}" - Debugf(configName, "detected overridden config - adding %q suffix to name", suffix) - // Add the suffix to the config name - // - // These need to work as filesystem names as the VFS cache will use them - configName += suffix - } - return fsInfo.NewFs(ctx, configName, fsPath, config) -} - -// ConfigString returns a canonical version of the config string used -// to configure the Fs as passed to fs.NewFs -func ConfigString(f Fs) string { - name := f.Name() - root := f.Root() - if name == "local" && f.Features().IsLocal { - return root - } - return name + ":" + root -} - -// TemporaryLocalFs creates a local FS in the OS's temporary directory. -// -// No cleanup is performed, the caller must call Purge on the Fs themselves. -func TemporaryLocalFs(ctx context.Context) (Fs, error) { - path, err := ioutil.TempDir("", "rclone-spool") - if err == nil { - err = os.Remove(path) - } - if err != nil { - return nil, err - } - path = filepath.ToSlash(path) - return NewFs(ctx, path) -} - // CheckClose is a utility function used to check the return from // Close in a defer statement. func CheckClose(c io.Closer, err *error) { @@ -1519,86 +89,3 @@ func GetModifyWindow(ctx context.Context, fss ...Info) time.Duration { } return window } - -// Pacer is a simple wrapper around a pacer.Pacer with logging. -type Pacer struct { - *pacer.Pacer -} - -type logCalculator struct { - pacer.Calculator -} - -// NewPacer creates a Pacer for the given Fs and Calculator. -func NewPacer(ctx context.Context, c pacer.Calculator) *Pacer { - ci := GetConfig(ctx) - retries := ci.LowLevelRetries - if retries <= 0 { - retries = 1 - } - p := &Pacer{ - Pacer: pacer.New( - pacer.InvokerOption(pacerInvoker), - pacer.MaxConnectionsOption(ci.Checkers+ci.Transfers), - pacer.RetriesOption(retries), - pacer.CalculatorOption(c), - ), - } - p.SetCalculator(c) - return p -} - -func (d *logCalculator) Calculate(state pacer.State) time.Duration { - oldSleepTime := state.SleepTime - newSleepTime := d.Calculator.Calculate(state) - if state.ConsecutiveRetries > 0 { - if newSleepTime != oldSleepTime { - Debugf("pacer", "Rate limited, increasing sleep to %v", newSleepTime) - } - } else { - if newSleepTime != oldSleepTime { - Debugf("pacer", "Reducing sleep to %v", newSleepTime) - } - } - return newSleepTime -} - -// SetCalculator sets the pacing algorithm. Don't modify the Calculator object -// afterwards, use the ModifyCalculator method when needed. -// -// It will choose the default algorithm if nil is passed in. -func (p *Pacer) SetCalculator(c pacer.Calculator) { - switch c.(type) { - case *logCalculator: - Logf("pacer", "Invalid Calculator in fs.Pacer.SetCalculator") - case nil: - c = &logCalculator{pacer.NewDefault()} - default: - c = &logCalculator{c} - } - - p.Pacer.SetCalculator(c) -} - -// ModifyCalculator calls the given function with the currently configured -// Calculator and the Pacer lock held. -func (p *Pacer) ModifyCalculator(f func(pacer.Calculator)) { - p.ModifyCalculator(func(c pacer.Calculator) { - switch _c := c.(type) { - case *logCalculator: - f(_c.Calculator) - default: - Logf("pacer", "Invalid Calculator in fs.Pacer: %t", c) - f(c) - } - }) -} - -func pacerInvoker(try, retries int, f pacer.Paced) (retry bool, err error) { - retry, err = f() - if retry { - Debugf("pacer", "low level retry %d/%d (error %v)", try, retries, err) - err = fserrors.RetryError(err) - } - return -} diff --git a/fs/newfs.go b/fs/newfs.go new file mode 100644 index 000000000..1ac5e4807 --- /dev/null +++ b/fs/newfs.go @@ -0,0 +1,122 @@ +// NewFs and its helpers + +package fs + +import ( + "context" + "crypto/md5" + "encoding/base64" + "io/ioutil" + "os" + "path/filepath" + "strings" + + "github.com/rclone/rclone/fs/config/configmap" + "github.com/rclone/rclone/fs/fspath" +) + +// NewFs makes a new Fs object from the path +// +// The path is of the form remote:path +// +// Remotes are looked up in the config file. If the remote isn't +// found then NotFoundInConfigFile will be returned. +// +// On Windows avoid single character remote names as they can be mixed +// up with drive letters. +func NewFs(ctx context.Context, path string) (Fs, error) { + Debugf(nil, "Creating backend with remote %q", path) + fsInfo, configName, fsPath, config, err := ConfigFs(path) + if err != nil { + return nil, err + } + overridden := fsInfo.Options.Overridden(config) + if len(overridden) > 0 { + extraConfig := overridden.String() + //Debugf(nil, "detected overriden config %q", extraConfig) + md5sumBinary := md5.Sum([]byte(extraConfig)) + suffix := base64.RawURLEncoding.EncodeToString(md5sumBinary[:]) + // 5 characters length is 5*6 = 30 bits of base64 + const maxLength = 5 + if len(suffix) > maxLength { + suffix = suffix[:maxLength] + } + suffix = "{" + suffix + "}" + Debugf(configName, "detected overridden config - adding %q suffix to name", suffix) + // Add the suffix to the config name + // + // These need to work as filesystem names as the VFS cache will use them + configName += suffix + } + return fsInfo.NewFs(ctx, configName, fsPath, config) +} + +// ConfigFs makes the config for calling NewFs with. +// +// It parses the path which is of the form remote:path +// +// Remotes are looked up in the config file. If the remote isn't +// found then NotFoundInConfigFile will be returned. +func ConfigFs(path string) (fsInfo *RegInfo, configName, fsPath string, config *configmap.Map, err error) { + // Parse the remote path + fsInfo, configName, fsPath, connectionStringConfig, err := ParseRemote(path) + if err != nil { + return + } + config = ConfigMap(fsInfo, configName, connectionStringConfig) + return +} + +// ParseRemote deconstructs a path into configName, fsPath, looking up +// the fsName in the config file (returning NotFoundInConfigFile if not found) +func ParseRemote(path string) (fsInfo *RegInfo, configName, fsPath string, connectionStringConfig configmap.Simple, err error) { + parsed, err := fspath.Parse(path) + if err != nil { + return nil, "", "", nil, err + } + configName, fsPath = parsed.Name, parsed.Path + var fsName string + var ok bool + if configName != "" { + if strings.HasPrefix(configName, ":") { + fsName = configName[1:] + } else { + m := ConfigMap(nil, configName, parsed.Config) + fsName, ok = m.Get("type") + if !ok { + return nil, "", "", nil, ErrorNotFoundInConfigFile + } + } + } else { + fsName = "local" + configName = "local" + } + fsInfo, err = Find(fsName) + return fsInfo, configName, fsPath, parsed.Config, err +} + +// ConfigString returns a canonical version of the config string used +// to configure the Fs as passed to fs.NewFs +func ConfigString(f Fs) string { + name := f.Name() + root := f.Root() + if name == "local" && f.Features().IsLocal { + return root + } + return name + ":" + root +} + +// TemporaryLocalFs creates a local FS in the OS's temporary directory. +// +// No cleanup is performed, the caller must call Purge on the Fs themselves. +func TemporaryLocalFs(ctx context.Context) (Fs, error) { + path, err := ioutil.TempDir("", "rclone-spool") + if err == nil { + err = os.Remove(path) + } + if err != nil { + return nil, err + } + path = filepath.ToSlash(path) + return NewFs(ctx, path) +} diff --git a/fs/options.go b/fs/open_options.go similarity index 99% rename from fs/options.go rename to fs/open_options.go index a85cd6f00..dbf857567 100644 --- a/fs/options.go +++ b/fs/open_options.go @@ -1,4 +1,4 @@ -// Define the options for Open +// Options for Open package fs diff --git a/fs/options_test.go b/fs/open_options_test.go similarity index 100% rename from fs/options_test.go rename to fs/open_options_test.go diff --git a/fs/pacer.go b/fs/pacer.go new file mode 100644 index 000000000..fbbfb7d63 --- /dev/null +++ b/fs/pacer.go @@ -0,0 +1,94 @@ +// Pacer with logging and calculator + +package fs + +import ( + "context" + "time" + + "github.com/rclone/rclone/fs/fserrors" + "github.com/rclone/rclone/lib/pacer" +) + +// Pacer is a simple wrapper around a pacer.Pacer with logging. +type Pacer struct { + *pacer.Pacer +} + +type logCalculator struct { + pacer.Calculator +} + +// NewPacer creates a Pacer for the given Fs and Calculator. +func NewPacer(ctx context.Context, c pacer.Calculator) *Pacer { + ci := GetConfig(ctx) + retries := ci.LowLevelRetries + if retries <= 0 { + retries = 1 + } + p := &Pacer{ + Pacer: pacer.New( + pacer.InvokerOption(pacerInvoker), + pacer.MaxConnectionsOption(ci.Checkers+ci.Transfers), + pacer.RetriesOption(retries), + pacer.CalculatorOption(c), + ), + } + p.SetCalculator(c) + return p +} + +func (d *logCalculator) Calculate(state pacer.State) time.Duration { + oldSleepTime := state.SleepTime + newSleepTime := d.Calculator.Calculate(state) + if state.ConsecutiveRetries > 0 { + if newSleepTime != oldSleepTime { + Debugf("pacer", "Rate limited, increasing sleep to %v", newSleepTime) + } + } else { + if newSleepTime != oldSleepTime { + Debugf("pacer", "Reducing sleep to %v", newSleepTime) + } + } + return newSleepTime +} + +// SetCalculator sets the pacing algorithm. Don't modify the Calculator object +// afterwards, use the ModifyCalculator method when needed. +// +// It will choose the default algorithm if nil is passed in. +func (p *Pacer) SetCalculator(c pacer.Calculator) { + switch c.(type) { + case *logCalculator: + Logf("pacer", "Invalid Calculator in fs.Pacer.SetCalculator") + case nil: + c = &logCalculator{pacer.NewDefault()} + default: + c = &logCalculator{c} + } + + p.Pacer.SetCalculator(c) +} + +// ModifyCalculator calls the given function with the currently configured +// Calculator and the Pacer lock held. +func (p *Pacer) ModifyCalculator(f func(pacer.Calculator)) { + p.ModifyCalculator(func(c pacer.Calculator) { + switch _c := c.(type) { + case *logCalculator: + f(_c.Calculator) + default: + Logf("pacer", "Invalid Calculator in fs.Pacer: %t", c) + f(c) + } + }) +} + +func pacerInvoker(try, retries int, f pacer.Paced) (retry bool, err error) { + retry, err = f() + if retry { + Debugf("pacer", "low level retry %d/%d (error %v)", try, retries, err) + err = fserrors.RetryError(err) + } + return +} diff --git a/fs/registry.go b/fs/registry.go new file mode 100644 index 000000000..24b4772bf --- /dev/null +++ b/fs/registry.go @@ -0,0 +1,277 @@ +// Filesystem registry and backend options + +package fs + +import ( + "context" + "encoding/json" + "fmt" + "log" + "reflect" + "sort" + "strings" + + "github.com/pkg/errors" + "github.com/rclone/rclone/fs/config/configmap" + "github.com/rclone/rclone/fs/config/configstruct" +) + +// Registry of filesystems +var Registry []*RegInfo + +// RegInfo provides information about a filesystem +type RegInfo struct { + // Name of this fs + Name string + // Description of this fs - defaults to Name + Description string + // Prefix for command line flags for this fs - defaults to Name if not set + Prefix string + // Create a new file system. If root refers to an existing + // object, then it should return an Fs which which points to + // the parent of that object and ErrorIsFile. + NewFs func(ctx context.Context, name string, root string, config configmap.Mapper) (Fs, error) `json:"-"` + // Function to call to help with config - see docs for ConfigIn for more info + Config func(ctx context.Context, name string, m configmap.Mapper, configIn ConfigIn) (*ConfigOut, error) `json:"-"` + // Options for the Fs configuration + Options Options + // The command help, if any + CommandHelp []CommandHelp +} + +// FileName returns the on disk file name for this backend +func (ri *RegInfo) FileName() string { + return strings.Replace(ri.Name, " ", "", -1) +} + +// Options is a slice of configuration Option for a backend +type Options []Option + +// Set the default values for the options +func (os Options) setValues() { + for i := range os { + o := &os[i] + if o.Default == nil { + o.Default = "" + } + } +} + +// Get the Option corresponding to name or return nil if not found +func (os Options) Get(name string) *Option { + for i := range os { + opt := &os[i] + if opt.Name == name { + return opt + } + } + return nil +} + +// Overridden discovers which config items have been overridden in the +// configmap passed in, either by the config string, command line +// flags or environment variables +func (os Options) Overridden(m *configmap.Map) configmap.Simple { + var overridden = configmap.Simple{} + for i := range os { + opt := &os[i] + value, isSet := m.GetPriority(opt.Name, configmap.PriorityNormal) + if isSet { + overridden.Set(opt.Name, value) + } + } + return overridden +} + +// NonDefault discovers which config values aren't at their default +func (os Options) NonDefault(m configmap.Getter) configmap.Simple { + var nonDefault = configmap.Simple{} + for i := range os { + opt := &os[i] + value, isSet := m.Get(opt.Name) + if !isSet { + continue + } + defaultValue := fmt.Sprint(opt.Default) + if value != defaultValue { + nonDefault.Set(opt.Name, value) + } + } + return nonDefault +} + +// HasAdvanced discovers if any options have an Advanced setting +func (os Options) HasAdvanced() bool { + for i := range os { + opt := &os[i] + if opt.Advanced { + return true + } + } + return false +} + +// OptionVisibility controls whether the options are visible in the +// configurator or the command line. +type OptionVisibility byte + +// Constants Option.Hide +const ( + OptionHideCommandLine OptionVisibility = 1 << iota + OptionHideConfigurator + OptionHideBoth = OptionHideCommandLine | OptionHideConfigurator +) + +// Option is describes an option for the config wizard +// +// This also describes command line options and environment variables +type Option struct { + Name string // name of the option in snake_case + Help string // Help, the first line only is used for the command line help + Provider string // Set to filter on provider + Default interface{} // default value, nil => "" + Value interface{} // value to be set by flags + Examples OptionExamples `json:",omitempty"` // config examples + ShortOpt string // the short option for this if required + Hide OptionVisibility // set this to hide the config from the configurator or the command line + Required bool // this option is required + IsPassword bool // set if the option is a password + NoPrefix bool // set if the option for this should not use the backend prefix + Advanced bool // set if this is an advanced config option + Exclusive bool // set if the answer can only be one of the examples +} + +// BaseOption is an alias for Option used internally +type BaseOption Option + +// MarshalJSON turns an Option into JSON +// +// It adds some generated fields for ease of use +// - DefaultStr - a string rendering of Default +// - ValueStr - a string rendering of Value +// - Type - the type of the option +func (o *Option) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + BaseOption + DefaultStr string + ValueStr string + Type string + }{ + BaseOption: BaseOption(*o), + DefaultStr: fmt.Sprint(o.Default), + ValueStr: o.String(), + Type: o.Type(), + }) +} + +// GetValue gets the current current value which is the default if not set +func (o *Option) GetValue() interface{} { + val := o.Value + if val == nil { + val = o.Default + if val == nil { + val = "" + } + } + return val +} + +// String turns Option into a string +func (o *Option) String() string { + return fmt.Sprint(o.GetValue()) +} + +// Set an Option from a string +func (o *Option) Set(s string) (err error) { + newValue, err := configstruct.StringToInterface(o.GetValue(), s) + if err != nil { + return err + } + o.Value = newValue + return nil +} + +// Type of the value +func (o *Option) Type() string { + return reflect.TypeOf(o.GetValue()).Name() +} + +// FlagName for the option +func (o *Option) FlagName(prefix string) string { + name := strings.Replace(o.Name, "_", "-", -1) // convert snake_case to kebab-case + if !o.NoPrefix { + name = prefix + "-" + name + } + return name +} + +// EnvVarName for the option +func (o *Option) EnvVarName(prefix string) string { + return OptionToEnv(prefix + "-" + o.Name) +} + +// Copy makes a shallow copy of the option +func (o *Option) Copy() *Option { + copy := new(Option) + *copy = *o + return copy +} + +// OptionExamples is a slice of examples +type OptionExamples []OptionExample + +// Len is part of sort.Interface. +func (os OptionExamples) Len() int { return len(os) } + +// Swap is part of sort.Interface. +func (os OptionExamples) Swap(i, j int) { os[i], os[j] = os[j], os[i] } + +// Less is part of sort.Interface. +func (os OptionExamples) Less(i, j int) bool { return os[i].Help < os[j].Help } + +// Sort sorts an OptionExamples +func (os OptionExamples) Sort() { sort.Sort(os) } + +// OptionExample describes an example for an Option +type OptionExample struct { + Value string + Help string + Provider string +} + +// Register a filesystem +// +// Fs modules should use this in an init() function +func Register(info *RegInfo) { + info.Options.setValues() + if info.Prefix == "" { + info.Prefix = info.Name + } + Registry = append(Registry, info) +} + +// Find looks for a RegInfo object for the name passed in. The name +// can be either the Name or the Prefix. +// +// Services are looked up in the config file +func Find(name string) (*RegInfo, error) { + for _, item := range Registry { + if item.Name == name || item.Prefix == name || item.FileName() == name { + return item, nil + } + } + return nil, errors.Errorf("didn't find backend called %q", name) +} + +// MustFind looks for an Info object for the type name passed in +// +// Services are looked up in the config file +// +// Exits with a fatal error if not found +func MustFind(name string) *RegInfo { + fs, err := Find(name) + if err != nil { + log.Fatalf("Failed to find remote: %v", err) + } + return fs +} diff --git a/fs/types.go b/fs/types.go new file mode 100644 index 000000000..a7cbba457 --- /dev/null +++ b/fs/types.go @@ -0,0 +1,266 @@ +// Filesystem related types and interfaces +// Note that optional interfaces are found in features.go + +package fs + +import ( + "context" + "io" + "time" + + "github.com/rclone/rclone/fs/hash" +) + +// Fs is the interface a cloud storage system must provide +type Fs interface { + Info + + // List the objects and directories in dir into entries. The + // entries can be returned in any order but should be for a + // complete directory. + // + // dir should be "" to list the root, and should not have + // trailing slashes. + // + // This should return ErrDirNotFound if the directory isn't + // found. + List(ctx context.Context, dir string) (entries DirEntries, err error) + + // NewObject finds the Object at remote. If it can't be found + // it returns the error ErrorObjectNotFound. + NewObject(ctx context.Context, remote string) (Object, error) + + // Put in to the remote path with the modTime given of the given size + // + // When called from outside an Fs by rclone, src.Size() will always be >= 0. + // But for unknown-sized objects (indicated by src.Size() == -1), Put should either + // return an error or upload it properly (rather than e.g. calling panic). + // + // May create the object even if it returns an error - if so + // will return the object and the error, otherwise will return + // nil and the error + Put(ctx context.Context, in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error) + + // Mkdir makes the directory (container, bucket) + // + // Shouldn't return an error if it already exists + Mkdir(ctx context.Context, dir string) error + + // Rmdir removes the directory (container, bucket) if empty + // + // Return an error if it doesn't exist or isn't empty + Rmdir(ctx context.Context, dir string) error +} + +// Info provides a read only interface to information about a filesystem. +type Info interface { + // Name of the remote (as passed into NewFs) + Name() string + + // Root of the remote (as passed into NewFs) + Root() string + + // String returns a description of the FS + String() string + + // Precision of the ModTimes in this Fs + Precision() time.Duration + + // Returns the supported hash types of the filesystem + Hashes() hash.Set + + // Features returns the optional features of this Fs + Features() *Features +} + +// Object is a filesystem like object provided by an Fs +type Object interface { + ObjectInfo + + // SetModTime sets the metadata on the object to set the modification date + SetModTime(ctx context.Context, t time.Time) error + + // Open opens the file for read. Call Close() on the returned io.ReadCloser + Open(ctx context.Context, options ...OpenOption) (io.ReadCloser, error) + + // Update in to the object with the modTime given of the given size + // + // When called from outside an Fs by rclone, src.Size() will always be >= 0. + // But for unknown-sized objects (indicated by src.Size() == -1), Upload should either + // return an error or update the object properly (rather than e.g. calling panic). + Update(ctx context.Context, in io.Reader, src ObjectInfo, options ...OpenOption) error + + // Removes this object + Remove(ctx context.Context) error +} + +// ObjectInfo provides read only information about an object. +type ObjectInfo interface { + DirEntry + + // Fs returns read only access to the Fs that this object is part of + Fs() Info + + // Hash returns the selected checksum of the file + // If no checksum is available it returns "" + Hash(ctx context.Context, ty hash.Type) (string, error) + + // Storable says whether this object can be stored + Storable() bool +} + +// DirEntry provides read only information about the common subset of +// a Dir or Object. These are returned from directory listings - type +// assert them into the correct type. +type DirEntry interface { + // String returns a description of the Object + String() string + + // Remote returns the remote path + Remote() string + + // ModTime returns the modification date of the file + // It should return a best guess if one isn't available + ModTime(context.Context) time.Time + + // Size returns the size of the file + Size() int64 +} + +// Directory is a filesystem like directory provided by an Fs +type Directory interface { + DirEntry + + // Items returns the count of items in this directory or this + // directory and subdirectories if known, -1 for unknown + Items() int64 + + // ID returns the internal ID of this directory if known, or + // "" otherwise + ID() string +} + +// MimeTyper is an optional interface for Object +type MimeTyper interface { + // MimeType returns the content type of the Object if + // known, or "" if not + MimeType(ctx context.Context) string +} + +// IDer is an optional interface for Object +type IDer interface { + // ID returns the ID of the Object if known, or "" if not + ID() string +} + +// ParentIDer is an optional interface for Object +type ParentIDer interface { + // ParentID returns the ID of the parent directory if known or nil if not + ParentID() string +} + +// ObjectUnWrapper is an optional interface for Object +type ObjectUnWrapper interface { + // UnWrap returns the Object that this Object is wrapping or + // nil if it isn't wrapping anything + UnWrap() Object +} + +// SetTierer is an optional interface for Object +type SetTierer interface { + // SetTier performs changing storage tier of the Object if + // multiple storage classes supported + SetTier(tier string) error +} + +// GetTierer is an optional interface for Object +type GetTierer interface { + // GetTier returns storage tier or class of the Object + GetTier() string +} + +// FullObjectInfo contains all the read-only optional interfaces +// +// Use for checking making wrapping ObjectInfos implement everything +type FullObjectInfo interface { + ObjectInfo + MimeTyper + IDer + ObjectUnWrapper + GetTierer +} + +// FullObject contains all the optional interfaces for Object +// +// Use for checking making wrapping Objects implement everything +type FullObject interface { + Object + MimeTyper + IDer + ObjectUnWrapper + GetTierer + SetTierer +} + +// ObjectOptionalInterfaces returns the names of supported and +// unsupported optional interfaces for an Object +func ObjectOptionalInterfaces(o Object) (supported, unsupported []string) { + store := func(ok bool, name string) { + if ok { + supported = append(supported, name) + } else { + unsupported = append(unsupported, name) + } + } + + _, ok := o.(MimeTyper) + store(ok, "MimeType") + + _, ok = o.(IDer) + store(ok, "ID") + + _, ok = o.(ObjectUnWrapper) + store(ok, "UnWrap") + + _, ok = o.(SetTierer) + store(ok, "SetTier") + + _, ok = o.(GetTierer) + store(ok, "GetTier") + + return supported, unsupported +} + +// ListRCallback defines a callback function for ListR to use +// +// It is called for each tranche of entries read from the listing and +// if it returns an error, the listing stops. +type ListRCallback func(entries DirEntries) error + +// ListRFn is defines the call used to recursively list a directory +type ListRFn func(ctx context.Context, dir string, callback ListRCallback) error + +// NewUsageValue makes a valid value +func NewUsageValue(value int64) *int64 { + p := new(int64) + *p = value + return p +} + +// Usage is returned by the About call +// +// If a value is nil then it isn't supported by that backend +type Usage struct { + Total *int64 `json:"total,omitempty"` // quota of bytes that can be used + Used *int64 `json:"used,omitempty"` // bytes in use + Trashed *int64 `json:"trashed,omitempty"` // bytes in trash + Other *int64 `json:"other,omitempty"` // other usage e.g. gmail in drive + Free *int64 `json:"free,omitempty"` // bytes which can be uploaded before reaching the quota + Objects *int64 `json:"objects,omitempty"` // objects in the storage system +} + +// WriterAtCloser wraps io.WriterAt and io.Closer +type WriterAtCloser interface { + io.WriterAt + io.Closer +}