From 1b92e4636e789f4007adec8fec04b93766ae5671 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 23 Feb 2021 10:34:58 +0000 Subject: [PATCH] rc: implement passing filter config with _filter parameter --- docs/content/rc.md | 33 +++++++++++++++++++++++++++++ fs/rc/jobs/job.go | 27 ++++++++++++++++++++++++ fs/rc/jobs/job_test.go | 48 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 108 insertions(+) diff --git a/docs/content/rc.md b/docs/content/rc.md index c2323b01d..ffb37369b 100644 --- a/docs/content/rc.md +++ b/docs/content/rc.md @@ -303,6 +303,39 @@ setting the equivalent of `--buffer-size` in string or integer format. If you wish to check the `_config` assignment has worked properly then calling `options/local` will show what the value got set to. +### Setting filter flags with _filter + +If you wish to set filters for the duration of an rc call only then +pass in the `_filter` parameter. + +This should be in the same format as the `filter` key returned by +[options/get](#options-get). + +For example, if you wished to run a sync with these flags + + --max-size 1M --max-age 42s --include "a" --include "b" + +you would pass this parameter in your JSON blob. + + "_filter":{"MaxSize":"1M", "IncludeRule":["a","b"], "MaxAge":"42s"} + +If using `rclone rc` this could be passed as + + rclone rc ... _filter='{"MaxSize":"1M", "IncludeRule":["a","b"], "MaxAge":"42s"}' + +Any filter parameters you don't set will inherit the global defaults +which were set with command line flags or environment variables. + +Note that it is possible to set some values as strings or integers - +see [data types](/#data-types) for more info. Here is an example +setting the equivalent of `--buffer-size` in string or integer format. + + "_filter":{"MinSize": "42M"} + "_filter":{"MinSize": 44040192} + +If you wish to check the `_filter` assignment has worked properly then +calling `options/local` will show what the value got set to. + ### Assigning operations to groups with _group = value Each rc call has its own stats group for tracking its metrics. By default diff --git a/fs/rc/jobs/job.go b/fs/rc/jobs/job.go index 9dbb34fa6..d1d5ae4cc 100644 --- a/fs/rc/jobs/job.go +++ b/fs/rc/jobs/job.go @@ -13,6 +13,7 @@ import ( "github.com/pkg/errors" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/accounting" + "github.com/rclone/rclone/fs/filter" "github.com/rclone/rclone/fs/rc" ) @@ -216,6 +217,27 @@ func getConfig(ctx context.Context, in rc.Params) (context.Context, error) { return ctx, nil } +// See if _filter is set and if so adjust ctx to include it +func getFilter(ctx context.Context, in rc.Params) (context.Context, error) { + if _, ok := in["_filter"]; !ok { + return ctx, nil + } + // Copy of the current filter options + opt := filter.GetConfig(ctx).Opt + // Update the options from the parameter + err := in.GetStruct("_filter", &opt) + if err != nil { + return ctx, err + } + fi, err := filter.NewFilter(&opt) + if err != nil { + return ctx, err + } + ctx = filter.ReplaceConfig(ctx, fi) + delete(in, "_filter") // remove the parameter + return ctx, nil +} + // NewJob creates a Job and executes it, possibly in the background if _async is set func (jobs *Jobs) NewJob(ctx context.Context, fn rc.Func, in rc.Params) (job *Job, out rc.Params, err error) { id := atomic.AddInt64(&jobID, 1) @@ -231,6 +253,11 @@ func (jobs *Jobs) NewJob(ctx context.Context, fn rc.Func, in rc.Params) (job *Jo return nil, nil, err } + ctx, err = getFilter(ctx, in) + if err != nil { + return nil, nil, err + } + ctx, group, err := getGroup(ctx, in, id) if err != nil { return nil, nil, err diff --git a/fs/rc/jobs/job_test.go b/fs/rc/jobs/job_test.go index 557509497..29cb35009 100644 --- a/fs/rc/jobs/job_test.go +++ b/fs/rc/jobs/job_test.go @@ -8,6 +8,8 @@ import ( "github.com/pkg/errors" "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/accounting" + "github.com/rclone/rclone/fs/filter" "github.com/rclone/rclone/fs/rc" "github.com/rclone/rclone/fs/rc/rcflags" "github.com/rclone/rclone/fstest/testy" @@ -252,9 +254,11 @@ func TestExecuteJob(t *testing.T) { func TestExecuteJobWithConfig(t *testing.T) { ctx := context.Background() jobID = 0 + called := false jobFn := func(ctx context.Context, in rc.Params) (rc.Params, error) { ci := fs.GetConfig(ctx) assert.Equal(t, 42*fs.MebiByte, ci.BufferSize) + called = true return nil, nil } _, _, err := NewJob(context.Background(), jobFn, rc.Params{ @@ -263,15 +267,59 @@ func TestExecuteJobWithConfig(t *testing.T) { }, }) require.NoError(t, err) + assert.Equal(t, true, called) + // Retest with string parameter jobID = 0 + called = false _, _, err = NewJob(ctx, jobFn, rc.Params{ "_config": `{"BufferSize": "42M"}`, }) require.NoError(t, err) + assert.Equal(t, true, called) + // Check that wasn't the default ci := fs.GetConfig(ctx) assert.NotEqual(t, 42*fs.MebiByte, ci.BufferSize) } +func TestExecuteJobWithFilter(t *testing.T) { + ctx := context.Background() + called := false + jobID = 0 + jobFn := func(ctx context.Context, in rc.Params) (rc.Params, error) { + fi := filter.GetConfig(ctx) + assert.Equal(t, fs.SizeSuffix(1024), fi.Opt.MaxSize) + assert.Equal(t, []string{"a", "b", "c"}, fi.Opt.IncludeRule) + called = true + return nil, nil + } + _, _, err := NewJob(ctx, jobFn, rc.Params{ + "_filter": rc.Params{ + "IncludeRule": []string{"a", "b", "c"}, + "MaxSize": "1k", + }, + }) + require.NoError(t, err) + assert.Equal(t, true, called) +} + +func TestExecuteJobWithGroup(t *testing.T) { + ctx := context.Background() + jobID = 0 + called := false + jobFn := func(ctx context.Context, in rc.Params) (rc.Params, error) { + called = true + group, found := accounting.StatsGroupFromContext(ctx) + assert.Equal(t, true, found) + assert.Equal(t, "myparty", group) + return nil, nil + } + _, _, err := NewJob(ctx, jobFn, rc.Params{ + "_group": "myparty", + }) + require.NoError(t, err) + assert.Equal(t, true, called) +} + func TestExecuteJobErrorPropagation(t *testing.T) { ctx := context.Background() jobID = 0