From ec2ea37ad217c5cf9212ac3fb0879fe58dfd0937 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Mon, 7 Aug 2017 21:10:03 +0100 Subject: [PATCH] fs: Add --disable flag to disable optional features - fixes #1551 Eg to disable server side copy use `--disable copy`, to see a list of what you can disable, `--disable help`. --- docs/content/docs.md | 21 +++++++++++++++++ docs/content/drive.md | 7 +++--- fs/config.go | 9 ++++++++ fs/fs.go | 46 ++++++++++++++++++++++++++++++++++-- fs/fs_test.go | 54 +++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 132 insertions(+), 5 deletions(-) create mode 100644 fs/fs_test.go diff --git a/docs/content/docs.md b/docs/content/docs.md index 5d998cf58..d260d43d5 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -365,6 +365,27 @@ connection to go through to a remote object storage system. It is Mode to run dedupe command in. One of `interactive`, `skip`, `first`, `newest`, `oldest`, `rename`. The default is `interactive`. See the dedupe command for more information as to what these options mean. +### --disable FEATURE,FEATURE,... ### + +This disables a comma separated list of optional features. For example +to disable server side move and server side copy use: + + --disable move,copy + +The features can be put in in any case. + +To see a list of which features can be disabled use: + + --disable help + +See the overview [features](/overview/#features) and +[optional features](/overview/#optional-features) to get an idea of +which feature does what. + +This flag can be useful for debugging and in exceptional circumstances +(eg Google Drive limiting the total volume of Server Side Copies to +100GB/day). + ### -n, --dry-run ### Do a trial run with no permanent changes. Use this to see what rclone diff --git a/docs/content/drive.md b/docs/content/drive.md index 949618f74..066d6bc83 100644 --- a/docs/content/drive.md +++ b/docs/content/drive.md @@ -275,9 +275,10 @@ limited to transferring about 2 files per second only. Individual files may be transferred much faster at 100s of MBytes/s but lots of small files can take a long time. -Server side copies are also subject to a separate rate limit. If -you see User rate limit exceeded errors, wait at least 24 hours and -retry. +Server side copies are also subject to a separate rate limit. If you +see User rate limit exceeded errors, wait at least 24 hours and retry. +You can disable server side copies with `--disable copy` to download +and upload the files if you prefer. ### Duplicated files ### diff --git a/fs/config.go b/fs/config.go index d829a0336..76044116d 100644 --- a/fs/config.go +++ b/fs/config.go @@ -100,6 +100,7 @@ var ( tpsLimit = Float64P("tpslimit", "", 0, "Limit HTTP transactions per second to this.") tpsLimitBurst = IntP("tpslimit-burst", "", 1, "Max burst of transactions for --tpslimit.") bindAddr = StringP("bind", "", "", "Local address to bind to for outgoing connections, IPv4, IPv6 or name.") + disableFeatures = StringP("disable", "", "", "Disable a comma separated list of features. Use help to see a list.") logLevel = LogLevelNotice statsLogLevel = LogLevelInfo bwLimit BwTimetable @@ -235,6 +236,7 @@ type ConfigInfo struct { TPSLimit float64 TPSLimitBurst int BindAddr net.IP + DisableFeatures []string } // Return the path to the configuration file @@ -410,6 +412,13 @@ func LoadConfig() { Config.BindAddr = addrs[0] } + if *disableFeatures != "" { + if *disableFeatures == "help" { + log.Fatalf("Possible backend features are: %s\n", strings.Join(new(Features).List(), ", ")) + } + Config.DisableFeatures = strings.Split(*disableFeatures, ",") + } + // Load configuration file. var err error ConfigPath, err = filepath.Abs(*configFile) diff --git a/fs/fs.go b/fs/fs.go index 6830ac1d4..826d08e12 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -8,8 +8,10 @@ import ( "math" "os" "path/filepath" + "reflect" "regexp" "sort" + "strings" "time" "github.com/pkg/errors" @@ -347,6 +349,46 @@ type Features struct { ListR ListRFn } +// Disable nil's out the named feature. If it isn't found then it +// will log a message. +func (ft *Features) Disable(name string) *Features { + v := reflect.ValueOf(ft).Elem() + vType := v.Type() + for i := 0; i < v.NumField(); i++ { + vName := vType.Field(i).Name + field := v.Field(i) + if strings.EqualFold(name, vName) { + if !field.CanSet() { + Errorf(nil, "Can't set Feature %q", name) + } else { + zero := reflect.Zero(field.Type()) + field.Set(zero) + Debugf(nil, "Reset feature %q", name) + } + } + } + return ft +} + +// List returns a slice of all the possible feature names +func (ft *Features) List() (out []string) { + v := reflect.ValueOf(ft).Elem() + vType := v.Type() + for i := 0; i < v.NumField(); i++ { + out = append(out, vType.Field(i).Name) + } + return out +} + +// DisableList nil's out the comma separated list of named features. +// If it isn't found then it will log a message. +func (ft *Features) DisableList(list []string) *Features { + for _, feature := range list { + ft.Disable(strings.TrimSpace(feature)) + } + return ft +} + // Fill fills in the function pointers in the Features struct from the // optional interfaces. It returns the original updated Features // struct passed in. @@ -387,7 +429,7 @@ func (ft *Features) Fill(f Fs) *Features { if do, ok := f.(ListRer); ok { ft.ListR = do.ListR } - return ft + return ft.DisableList(Config.DisableFeatures) } // Mask the Features with the Fs passed in @@ -438,7 +480,7 @@ func (ft *Features) Mask(f Fs) *Features { if mask.ListR == nil { ft.ListR = nil } - return ft + return ft.DisableList(Config.DisableFeatures) } // Wrap makes a Copy of the features passed in, overriding the UnWrap diff --git a/fs/fs_test.go b/fs/fs_test.go new file mode 100644 index 000000000..10b0fe84f --- /dev/null +++ b/fs/fs_test.go @@ -0,0 +1,54 @@ +package fs + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestFeaturesDisable(t *testing.T) { + ft := new(Features) + ft.Copy = func(src Object, remote string) (Object, error) { + return nil, nil + } + ft.CaseInsensitive = true + + assert.NotNil(t, ft.Copy) + assert.Nil(t, ft.Purge) + ft.Disable("copy") + assert.Nil(t, ft.Copy) + assert.Nil(t, ft.Purge) + + assert.True(t, ft.CaseInsensitive) + assert.False(t, ft.DuplicateFiles) + ft.Disable("caseinsensitive") + assert.False(t, ft.CaseInsensitive) + assert.False(t, ft.DuplicateFiles) +} + +func TestFeaturesList(t *testing.T) { + ft := new(Features) + names := strings.Join(ft.List(), ",") + assert.True(t, strings.Contains(names, ",Copy,")) +} + +func TestFeaturesDisableList(t *testing.T) { + ft := new(Features) + ft.Copy = func(src Object, remote string) (Object, error) { + return nil, nil + } + ft.CaseInsensitive = true + + assert.NotNil(t, ft.Copy) + assert.Nil(t, ft.Purge) + assert.True(t, ft.CaseInsensitive) + assert.False(t, ft.DuplicateFiles) + + ft.DisableList([]string{"copy", "caseinsensitive"}) + + assert.Nil(t, ft.Copy) + assert.Nil(t, ft.Purge) + assert.False(t, ft.CaseInsensitive) + assert.False(t, ft.DuplicateFiles) +}