diff --git a/backend/alias/alias.go b/backend/alias/alias.go index 873320e98..283ab1c09 100644 --- a/backend/alias/alias.go +++ b/backend/alias/alias.go @@ -1,6 +1,7 @@ package alias import ( + "context" "errors" "strings" @@ -34,7 +35,7 @@ type Options struct { // NewFs constructs an Fs from the path. // // The returned Fs is the actual Fs, referenced by remote in the config -func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { +func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { // Parse config into Options struct opt := new(Options) err := configstruct.Set(m, opt) @@ -47,5 +48,5 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { if strings.HasPrefix(opt.Remote, name+":") { return nil, errors.New("can't point alias remote at itself - check the value of the remote setting") } - return cache.Get(fspath.JoinRootPath(opt.Remote, root)) + return cache.Get(ctx, fspath.JoinRootPath(opt.Remote, root)) } diff --git a/backend/alias/alias_internal_test.go b/backend/alias/alias_internal_test.go index 857bf1a37..46542c5f4 100644 --- a/backend/alias/alias_internal_test.go +++ b/backend/alias/alias_internal_test.go @@ -68,7 +68,7 @@ func TestNewFS(t *testing.T) { remoteRoot, err := filepath.Abs(filepath.FromSlash(path.Join("test/files", test.remoteRoot))) require.NoError(t, err, what) prepare(t, remoteRoot) - f, err := fs.NewFs(fmt.Sprintf("%s:%s", remoteName, test.fsRoot)) + f, err := fs.NewFs(context.Background(), fmt.Sprintf("%s:%s", remoteName, test.fsRoot)) require.NoError(t, err, what) gotEntries, err := f.List(context.Background(), test.fsList) require.NoError(t, err, what) @@ -90,7 +90,7 @@ func TestNewFS(t *testing.T) { func TestNewFSNoRemote(t *testing.T) { prepare(t, "") - f, err := fs.NewFs(fmt.Sprintf("%s:", remoteName)) + f, err := fs.NewFs(context.Background(), fmt.Sprintf("%s:", remoteName)) require.Error(t, err) require.Nil(t, f) @@ -98,7 +98,7 @@ func TestNewFSNoRemote(t *testing.T) { func TestNewFSInvalidRemote(t *testing.T) { prepare(t, "not_existing_test_remote:") - f, err := fs.NewFs(fmt.Sprintf("%s:", remoteName)) + f, err := fs.NewFs(context.Background(), fmt.Sprintf("%s:", remoteName)) require.Error(t, err) require.Nil(t, f) diff --git a/backend/amazonclouddrive/amazonclouddrive.go b/backend/amazonclouddrive/amazonclouddrive.go index 46cc02342..6a19fba60 100644 --- a/backend/amazonclouddrive/amazonclouddrive.go +++ b/backend/amazonclouddrive/amazonclouddrive.go @@ -239,8 +239,7 @@ func filterRequest(req *http.Request) { } // NewFs constructs an Fs from the path, container:path -func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { - ctx := context.Background() +func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { // Parse config into Options struct opt := new(Options) err := configstruct.Set(m, opt) diff --git a/backend/azureblob/azureblob.go b/backend/azureblob/azureblob.go index e62eea111..2de4f5e1f 100644 --- a/backend/azureblob/azureblob.go +++ b/backend/azureblob/azureblob.go @@ -379,8 +379,7 @@ func (f *Fs) setRoot(root string) { } // NewFs constructs an Fs from the path, container:path -func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { - ctx := context.Background() +func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { // Parse config into Options struct opt := new(Options) err := configstruct.Set(m, opt) diff --git a/backend/b2/b2.go b/backend/b2/b2.go index be32640ed..713653372 100644 --- a/backend/b2/b2.go +++ b/backend/b2/b2.go @@ -391,8 +391,7 @@ func (f *Fs) setRoot(root string) { } // NewFs constructs an Fs from the path, bucket:path -func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { - ctx := context.Background() +func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { // Parse config into Options struct opt := new(Options) err := configstruct.Set(m, opt) diff --git a/backend/box/box.go b/backend/box/box.go index c78219bc6..6a35fbcbb 100644 --- a/backend/box/box.go +++ b/backend/box/box.go @@ -372,8 +372,7 @@ func errorHandler(resp *http.Response) error { } // NewFs constructs an Fs from the path, container:path -func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { - ctx := context.Background() +func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { // Parse config into Options struct opt := new(Options) err := configstruct.Set(m, opt) diff --git a/backend/cache/cache.go b/backend/cache/cache.go index 44b788ca8..c103daa4d 100644 --- a/backend/cache/cache.go +++ b/backend/cache/cache.go @@ -340,7 +340,7 @@ func parseRootPath(path string) (string, error) { } // NewFs constructs an Fs from the path, container:path -func NewFs(name, rootPath string, m configmap.Mapper) (fs.Fs, error) { +func NewFs(ctx context.Context, name, rootPath string, m configmap.Mapper) (fs.Fs, error) { // Parse config into Options struct opt := new(Options) err := configstruct.Set(m, opt) @@ -362,7 +362,7 @@ func NewFs(name, rootPath string, m configmap.Mapper) (fs.Fs, error) { } remotePath := fspath.JoinRootPath(opt.Remote, rootPath) - wrappedFs, wrapErr := cache.Get(remotePath) + wrappedFs, wrapErr := cache.Get(ctx, remotePath) if wrapErr != nil && wrapErr != fs.ErrorIsFile { return nil, errors.Wrapf(wrapErr, "failed to make remote %q to wrap", remotePath) } @@ -479,7 +479,7 @@ func NewFs(name, rootPath string, m configmap.Mapper) (fs.Fs, error) { return nil, errors.Wrapf(err, "failed to create cache directory %v", f.opt.TempWritePath) } f.opt.TempWritePath = filepath.ToSlash(f.opt.TempWritePath) - f.tempFs, err = cache.Get(f.opt.TempWritePath) + f.tempFs, err = cache.Get(ctx, f.opt.TempWritePath) if err != nil { return nil, errors.Wrapf(err, "failed to create temp fs: %v", err) } @@ -506,7 +506,7 @@ func NewFs(name, rootPath string, m configmap.Mapper) (fs.Fs, error) { if doChangeNotify := wrappedFs.Features().ChangeNotify; doChangeNotify != nil { pollInterval := make(chan time.Duration, 1) pollInterval <- time.Duration(f.opt.ChunkCleanInterval) - doChangeNotify(context.Background(), f.receiveChangeNotify, pollInterval) + doChangeNotify(ctx, f.receiveChangeNotify, pollInterval) } f.features = (&fs.Features{ diff --git a/backend/cache/cache_internal_test.go b/backend/cache/cache_internal_test.go index 515849a26..4d8e8643b 100644 --- a/backend/cache/cache_internal_test.go +++ b/backend/cache/cache_internal_test.go @@ -932,7 +932,7 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool boltDb.PurgeTempUploads() _ = os.RemoveAll(path.Join(runInstance.tmpUploadDir, id)) } - f, err := cache.NewFs(remote, id, m) + f, err := cache.NewFs(context.Background(), remote, id, m) require.NoError(t, err) cfs, err := r.getCacheFs(f) require.NoError(t, err) diff --git a/backend/chunker/chunker.go b/backend/chunker/chunker.go index ed01ecbdc..c489da96f 100644 --- a/backend/chunker/chunker.go +++ b/backend/chunker/chunker.go @@ -223,7 +223,7 @@ It has the following fields: ver, size, nchunks, md5, sha1.`, } // NewFs constructs an Fs from the path, container:path -func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) { +func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs, error) { // Parse config into Options struct opt := new(Options) err := configstruct.Set(m, opt) @@ -248,7 +248,7 @@ func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) { } // Look for a file first remotePath := fspath.JoinRootPath(basePath, rpath) - baseFs, err := cache.Get(baseName + remotePath) + baseFs, err := cache.Get(ctx, baseName+remotePath) if err != fs.ErrorIsFile && err != nil { return nil, errors.Wrapf(err, "failed to make remote %q to wrap", baseName+remotePath) } @@ -276,7 +276,7 @@ func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) { // (yet can't satisfy fstest.CheckListing, will ignore) if err == nil && !f.useMeta && strings.Contains(rpath, "/") { firstChunkPath := f.makeChunkName(remotePath, 0, "", "") - _, testErr := cache.Get(baseName + firstChunkPath) + _, testErr := cache.Get(ctx, baseName+firstChunkPath) if testErr == fs.ErrorIsFile { err = testErr } diff --git a/backend/crypt/crypt.go b/backend/crypt/crypt.go index f94de0a49..ec583e52d 100644 --- a/backend/crypt/crypt.go +++ b/backend/crypt/crypt.go @@ -144,7 +144,7 @@ func NewCipher(m configmap.Mapper) (*Cipher, error) { } // NewFs constructs an Fs from the path, container:path -func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) { +func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs, error) { // Parse config into Options struct opt := new(Options) err := configstruct.Set(m, opt) @@ -166,14 +166,14 @@ func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) { // Look for a file first var wrappedFs fs.Fs if rpath == "" { - wrappedFs, err = cache.Get(remote) + wrappedFs, err = cache.Get(ctx, remote) } else { remotePath := fspath.JoinRootPath(remote, cipher.EncryptFileName(rpath)) - wrappedFs, err = cache.Get(remotePath) + wrappedFs, err = cache.Get(ctx, remotePath) // if that didn't produce a file, look for a directory if err != fs.ErrorIsFile { remotePath = fspath.JoinRootPath(remote, cipher.EncryptDirName(rpath)) - wrappedFs, err = cache.Get(remotePath) + wrappedFs, err = cache.Get(ctx, remotePath) } } if err != fs.ErrorIsFile && err != nil { diff --git a/backend/crypt/crypt_internal_test.go b/backend/crypt/crypt_internal_test.go index 554c6c57b..2f7f1f8b3 100644 --- a/backend/crypt/crypt_internal_test.go +++ b/backend/crypt/crypt_internal_test.go @@ -33,7 +33,7 @@ func (o testWrapper) UnWrap() fs.Object { // Create a temporary local fs to upload things from func makeTempLocalFs(t *testing.T) (localFs fs.Fs, cleanup func()) { - localFs, err := fs.TemporaryLocalFs() + localFs, err := fs.TemporaryLocalFs(context.Background()) require.NoError(t, err) cleanup = func() { require.NoError(t, localFs.Rmdir(context.Background(), "")) diff --git a/backend/drive/drive.go b/backend/drive/drive.go index 570526e60..a198c373d 100755 --- a/backend/drive/drive.go +++ b/backend/drive/drive.go @@ -1129,8 +1129,7 @@ func newFs(name, path string, m configmap.Mapper) (*Fs, error) { } // NewFs constructs an Fs from the path, container:path -func NewFs(name, path string, m configmap.Mapper) (fs.Fs, error) { - ctx := context.Background() +func NewFs(ctx context.Context, name, path string, m configmap.Mapper) (fs.Fs, error) { f, err := newFs(name, path, m) if err != nil { return nil, err @@ -3000,7 +2999,7 @@ func (f *Fs) copyID(ctx context.Context, id, dest string) (err error) { if destLeaf == "" { destLeaf = info.Name } - dstFs, err := cache.Get(destDir) + dstFs, err := cache.Get(ctx, destDir) if err != nil { return err } @@ -3187,7 +3186,7 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str dstFs := f target, ok := opt["target"] if ok { - targetFs, err := cache.Get(target) + targetFs, err := cache.Get(ctx, target) if err != nil { return nil, errors.Wrap(err, "couldn't find target") } diff --git a/backend/drive/drive_internal_test.go b/backend/drive/drive_internal_test.go index 4a1423eb0..e78a717a6 100644 --- a/backend/drive/drive_internal_test.go +++ b/backend/drive/drive_internal_test.go @@ -196,7 +196,7 @@ func (f *Fs) InternalTestDocumentImport(t *testing.T) { testFilesPath, err := filepath.Abs(filepath.FromSlash("test/files")) require.NoError(t, err) - testFilesFs, err := fs.NewFs(testFilesPath) + testFilesFs, err := fs.NewFs(context.Background(), testFilesPath) require.NoError(t, err) _, f.importMimeTypes, err = parseExtensions("odt,ods,doc") @@ -210,7 +210,7 @@ func (f *Fs) InternalTestDocumentUpdate(t *testing.T) { testFilesPath, err := filepath.Abs(filepath.FromSlash("test/files")) require.NoError(t, err) - testFilesFs, err := fs.NewFs(testFilesPath) + testFilesFs, err := fs.NewFs(context.Background(), testFilesPath) require.NoError(t, err) _, f.importMimeTypes, err = parseExtensions("odt,ods,doc") diff --git a/backend/dropbox/dropbox.go b/backend/dropbox/dropbox.go index 9a51b3c5c..411387e62 100755 --- a/backend/dropbox/dropbox.go +++ b/backend/dropbox/dropbox.go @@ -291,7 +291,7 @@ func (f *Fs) setUploadChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) } // NewFs constructs an Fs from the path, container:path -func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { +func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { // Parse config into Options struct opt := new(Options) err := configstruct.Set(m, opt) diff --git a/backend/fichier/fichier.go b/backend/fichier/fichier.go index 31f17088a..0f000af29 100644 --- a/backend/fichier/fichier.go +++ b/backend/fichier/fichier.go @@ -167,7 +167,7 @@ func (f *Fs) Features() *fs.Features { // // On Windows avoid single character remote names as they can be mixed // up with drive letters. -func NewFs(name string, root string, config configmap.Mapper) (fs.Fs, error) { +func NewFs(ctx context.Context, name string, root string, config configmap.Mapper) (fs.Fs, error) { opt := new(Options) err := configstruct.Set(config, opt) if err != nil { @@ -203,8 +203,6 @@ func NewFs(name string, root string, config configmap.Mapper) (fs.Fs, error) { f.dirCache = dircache.New(root, rootID, f) - ctx := context.Background() - // Find the current root err = f.dirCache.FindRoot(ctx, false) if err != nil { diff --git a/backend/ftp/ftp.go b/backend/ftp/ftp.go index dd21fdc49..df7d4dcf5 100644 --- a/backend/ftp/ftp.go +++ b/backend/ftp/ftp.go @@ -300,8 +300,7 @@ func (f *Fs) putFtpConnection(pc **ftp.ServerConn, err error) { } // NewFs constructs an Fs from the path, container:path -func NewFs(name, root string, m configmap.Mapper) (ff fs.Fs, err error) { - ctx := context.Background() +func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (ff fs.Fs, err error) { // defer fs.Trace(nil, "name=%q, root=%q", name, root)("fs=%v, err=%v", &ff, &err) // Parse config into Options struct opt := new(Options) diff --git a/backend/googlecloudstorage/googlecloudstorage.go b/backend/googlecloudstorage/googlecloudstorage.go index a2e143f04..a0cc41ce5 100644 --- a/backend/googlecloudstorage/googlecloudstorage.go +++ b/backend/googlecloudstorage/googlecloudstorage.go @@ -386,8 +386,7 @@ func (f *Fs) setRoot(root string) { } // NewFs constructs an Fs from the path, bucket:path -func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { - ctx := context.TODO() +func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { var oAuthClient *http.Client // Parse config into Options struct diff --git a/backend/googlephotos/googlephotos.go b/backend/googlephotos/googlephotos.go index a7674a2b5..603f81641 100644 --- a/backend/googlephotos/googlephotos.go +++ b/backend/googlephotos/googlephotos.go @@ -246,7 +246,7 @@ func errorHandler(resp *http.Response) error { } // NewFs constructs an Fs from the path, bucket:path -func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { +func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { // Parse config into Options struct opt := new(Options) err := configstruct.Set(m, opt) @@ -288,7 +288,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { var leaf string f.root, leaf = path.Split(f.root) f.root = strings.TrimRight(f.root, "/") - _, err := f.NewObject(context.TODO(), leaf) + _, err := f.NewObject(ctx, leaf) if err == nil { return f, fs.ErrorIsFile } diff --git a/backend/googlephotos/googlephotos_test.go b/backend/googlephotos/googlephotos_test.go index 65e811346..c65a02700 100644 --- a/backend/googlephotos/googlephotos_test.go +++ b/backend/googlephotos/googlephotos_test.go @@ -35,14 +35,14 @@ func TestIntegration(t *testing.T) { if *fstest.RemoteName == "" { *fstest.RemoteName = "TestGooglePhotos:" } - f, err := fs.NewFs(*fstest.RemoteName) + f, err := fs.NewFs(ctx, *fstest.RemoteName) if err == fs.ErrorNotFoundInConfigFile { t.Skip(fmt.Sprintf("Couldn't create google photos backend - skipping tests: %v", err)) } require.NoError(t, err) // Create local Fs pointing at testfiles - localFs, err := fs.NewFs("testfiles") + localFs, err := fs.NewFs(ctx, "testfiles") require.NoError(t, err) t.Run("CreateAlbum", func(t *testing.T) { @@ -155,7 +155,7 @@ func TestIntegration(t *testing.T) { }) t.Run("NewFsIsFile", func(t *testing.T) { - fNew, err := fs.NewFs(*fstest.RemoteName + remote) + fNew, err := fs.NewFs(ctx, *fstest.RemoteName+remote) assert.Equal(t, fs.ErrorIsFile, err) leaf := path.Base(remote) o, err := fNew.NewObject(ctx, leaf) diff --git a/backend/http/http.go b/backend/http/http.go index 884b9375d..1b342c84a 100644 --- a/backend/http/http.go +++ b/backend/http/http.go @@ -145,8 +145,7 @@ func statusError(res *http.Response, err error) error { // NewFs creates a new Fs object from the name and root. It connects to // the host specified in the config file. -func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { - ctx := context.TODO() +func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { // Parse config into Options struct opt := new(Options) err := configstruct.Set(m, opt) diff --git a/backend/http/http_internal_test.go b/backend/http/http_internal_test.go index e62a02b98..92017a389 100644 --- a/backend/http/http_internal_test.go +++ b/backend/http/http_internal_test.go @@ -69,7 +69,7 @@ func prepare(t *testing.T) (fs.Fs, func()) { m, tidy := prepareServer(t) // Instantiate it - f, err := NewFs(remoteName, "", m) + f, err := NewFs(context.Background(), remoteName, "", m) require.NoError(t, err) return f, tidy @@ -214,7 +214,7 @@ func TestIsAFileRoot(t *testing.T) { m, tidy := prepareServer(t) defer tidy() - f, err := NewFs(remoteName, "one%.txt", m) + f, err := NewFs(context.Background(), remoteName, "one%.txt", m) assert.Equal(t, err, fs.ErrorIsFile) testListRoot(t, f, false) @@ -224,7 +224,7 @@ func TestIsAFileSubDir(t *testing.T) { m, tidy := prepareServer(t) defer tidy() - f, err := NewFs(remoteName, "three/underthree.txt", m) + f, err := NewFs(context.Background(), remoteName, "three/underthree.txt", m) assert.Equal(t, err, fs.ErrorIsFile) entries, err := f.List(context.Background(), "") diff --git a/backend/hubic/hubic.go b/backend/hubic/hubic.go index 818586e24..6d1956ca9 100644 --- a/backend/hubic/hubic.go +++ b/backend/hubic/hubic.go @@ -146,7 +146,7 @@ func (f *Fs) getCredentials(ctx context.Context) (err error) { } // NewFs constructs an Fs from the path, container:path -func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { +func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { client, _, err := oauthutil.NewClient(name, m, oauthConfig) if err != nil { return nil, errors.Wrap(err, "failed to configure Hubic") @@ -176,7 +176,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { } // Make inner swift Fs from the connection - swiftFs, err := swift.NewFsWithConnection(opt, name, root, c, true) + swiftFs, err := swift.NewFsWithConnection(ctx, opt, name, root, c, true) if err != nil && err != fs.ErrorIsFile { return nil, err } diff --git a/backend/jottacloud/jottacloud.go b/backend/jottacloud/jottacloud.go index d3a3b92e7..9bf688d63 100644 --- a/backend/jottacloud/jottacloud.go +++ b/backend/jottacloud/jottacloud.go @@ -641,8 +641,7 @@ func grantTypeFilter(req *http.Request) { } // NewFs constructs an Fs from the path, container:path -func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { - ctx := context.TODO() +func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { // Parse config into Options struct opt := new(Options) err := configstruct.Set(m, opt) diff --git a/backend/koofr/koofr.go b/backend/koofr/koofr.go index f51368513..4775fc696 100644 --- a/backend/koofr/koofr.go +++ b/backend/koofr/koofr.go @@ -256,7 +256,7 @@ func (f *Fs) fullPath(part string) string { } // NewFs constructs a new filesystem given a root path and configuration options -func NewFs(name, root string, m configmap.Mapper) (ff fs.Fs, err error) { +func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (ff fs.Fs, err error) { opt := new(Options) err = configstruct.Set(m, opt) if err != nil { diff --git a/backend/local/local.go b/backend/local/local.go index 511c89c35..91eb4dbae 100644 --- a/backend/local/local.go +++ b/backend/local/local.go @@ -217,7 +217,7 @@ type Object struct { var errLinksAndCopyLinks = errors.New("can't use -l/--links with -L/--copy-links") // NewFs constructs an Fs from the path -func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { +func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { // Parse config into Options struct opt := new(Options) err := configstruct.Set(m, opt) diff --git a/backend/local/local_internal_test.go b/backend/local/local_internal_test.go index f5e651398..65b85712b 100644 --- a/backend/local/local_internal_test.go +++ b/backend/local/local_internal_test.go @@ -163,6 +163,6 @@ func TestSymlinkError(t *testing.T) { "links": "true", "copy_links": "true", } - _, err := NewFs("local", "/", m) + _, err := NewFs(context.Background(), "local", "/", m) assert.Equal(t, errLinksAndCopyLinks, err) } diff --git a/backend/mailru/mailru.go b/backend/mailru/mailru.go index 31b965c6c..9c51ac075 100644 --- a/backend/mailru/mailru.go +++ b/backend/mailru/mailru.go @@ -294,9 +294,8 @@ type Fs struct { } // NewFs constructs an Fs from the path, container:path -func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { +func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { // fs.Debugf(nil, ">>> NewFs %q %q", name, root) - ctx := context.Background() // Note: NewFs does not pass context! // Parse config into Options struct opt := new(Options) @@ -1662,7 +1661,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op // Attempt to put by hash using a spool file if trySpeedup { - tmpFs, err := fs.TemporaryLocalFs() + tmpFs, err := fs.TemporaryLocalFs(ctx) if err != nil { fs.Infof(tmpFs, "Failed to create spool FS: %v", err) } else { diff --git a/backend/mega/mega.go b/backend/mega/mega.go index b3f7ed54f..91fac4ce0 100644 --- a/backend/mega/mega.go +++ b/backend/mega/mega.go @@ -180,7 +180,7 @@ func (f *Fs) readMetaDataForPath(remote string) (info *mega.Node, err error) { } // NewFs constructs an Fs from the path, container:path -func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { +func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { // Parse config into Options struct opt := new(Options) err := configstruct.Set(m, opt) diff --git a/backend/memory/memory.go b/backend/memory/memory.go index 96df12160..5a5c9510e 100644 --- a/backend/memory/memory.go +++ b/backend/memory/memory.go @@ -222,7 +222,7 @@ func (f *Fs) setRoot(root string) { } // NewFs constructs an Fs from the path, bucket:path -func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { +func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { // Parse config into Options struct opt := new(Options) err := configstruct.Set(m, opt) diff --git a/backend/onedrive/onedrive.go b/backend/onedrive/onedrive.go index 8f3f5e69d..1697ad6f1 100755 --- a/backend/onedrive/onedrive.go +++ b/backend/onedrive/onedrive.go @@ -596,8 +596,7 @@ func (f *Fs) setUploadChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) } // NewFs constructs an Fs from the path, container:path -func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { - ctx := context.Background() +func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { // Parse config into Options struct opt := new(Options) err := configstruct.Set(m, opt) diff --git a/backend/opendrive/opendrive.go b/backend/opendrive/opendrive.go index 4c7dd3f7d..c71e7e888 100644 --- a/backend/opendrive/opendrive.go +++ b/backend/opendrive/opendrive.go @@ -164,8 +164,7 @@ func (f *Fs) DirCacheFlush() { } // NewFs constructs an Fs from the path, bucket:path -func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { - ctx := context.Background() +func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { // Parse config into Options struct opt := new(Options) err := configstruct.Set(m, opt) diff --git a/backend/pcloud/pcloud.go b/backend/pcloud/pcloud.go index d12410954..f6d42dd26 100644 --- a/backend/pcloud/pcloud.go +++ b/backend/pcloud/pcloud.go @@ -280,8 +280,7 @@ func errorHandler(resp *http.Response) error { } // NewFs constructs an Fs from the path, container:path -func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { - ctx := context.Background() +func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { // Parse config into Options struct opt := new(Options) err := configstruct.Set(m, opt) diff --git a/backend/premiumizeme/premiumizeme.go b/backend/premiumizeme/premiumizeme.go index c4e6b9988..a6df563c2 100644 --- a/backend/premiumizeme/premiumizeme.go +++ b/backend/premiumizeme/premiumizeme.go @@ -234,8 +234,7 @@ func (f *Fs) baseParams() url.Values { } // NewFs constructs an Fs from the path, container:path -func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { - ctx := context.Background() +func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { // Parse config into Options struct opt := new(Options) err := configstruct.Set(m, opt) diff --git a/backend/putio/fs.go b/backend/putio/fs.go index 0f24c65a9..0d2b87478 100644 --- a/backend/putio/fs.go +++ b/backend/putio/fs.go @@ -68,7 +68,7 @@ func parsePath(path string) (root string) { } // NewFs constructs an Fs from the path, container:path -func NewFs(name, root string, m configmap.Mapper) (f fs.Fs, err error) { +func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (f fs.Fs, err error) { // defer log.Trace(name, "root=%v", root)("f=%+v, err=%v", &f, &err) // Parse config into Options struct opt := new(Options) @@ -97,7 +97,6 @@ func NewFs(name, root string, m configmap.Mapper) (f fs.Fs, err error) { CanHaveEmptyDirectories: true, }).Fill(p) p.dirCache = dircache.New(root, "0", p) - ctx := context.Background() // Find the current root err = p.dirCache.FindRoot(ctx, false) if err != nil { diff --git a/backend/qingstor/qingstor.go b/backend/qingstor/qingstor.go index ce5fb9665..88387db95 100644 --- a/backend/qingstor/qingstor.go +++ b/backend/qingstor/qingstor.go @@ -319,7 +319,7 @@ func (f *Fs) setRoot(root string) { } // NewFs constructs an Fs from the path, bucket:path -func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { +func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { // Parse config into Options struct opt := new(Options) err := configstruct.Set(m, opt) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 2086eb36c..75511371f 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -1569,7 +1569,7 @@ func (f *Fs) setRoot(root string) { } // NewFs constructs an Fs from the path, bucket:path -func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { +func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { // Parse config into Options struct opt := new(Options) err := configstruct.Set(m, opt) diff --git a/backend/seafile/seafile.go b/backend/seafile/seafile.go index 628f89240..ee0e35668 100644 --- a/backend/seafile/seafile.go +++ b/backend/seafile/seafile.go @@ -147,7 +147,7 @@ type Fs struct { // ------------------------------------------------------------ // NewFs constructs an Fs from the path, container:path -func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { +func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { // Parse config into Options struct opt := new(Options) err := configstruct.Set(m, opt) @@ -205,7 +205,6 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { BucketBased: opt.LibraryName == "", }).Fill(f) - ctx := context.Background() serverInfo, err := f.getServerInfo(ctx) if err != nil { return nil, err diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go index bc7b55f40..ee6a6892c 100644 --- a/backend/sftp/sftp.go +++ b/backend/sftp/sftp.go @@ -413,12 +413,11 @@ func (f *Fs) putSftpConnection(pc **conn, err error) { // NewFs creates a new Fs object from the name and root. It connects to // the host specified in the config file. -func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { +func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { // This will hold the Fs object. We need to create it here // so we can refer to it in the SSH callback, but it's populated // in NewFsWithConnection f := &Fs{} - ctx := context.Background() // Parse config into Options struct opt := new(Options) err := configstruct.Set(m, opt) diff --git a/backend/sharefile/sharefile.go b/backend/sharefile/sharefile.go index b599bddec..252c8f129 100644 --- a/backend/sharefile/sharefile.go +++ b/backend/sharefile/sharefile.go @@ -410,8 +410,7 @@ func (f *Fs) setUploadCutoff(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) { } // NewFs constructs an Fs from the path, container:path -func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { - ctx := context.Background() +func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { // Parse config into Options struct opt := new(Options) err := configstruct.Set(m, opt) diff --git a/backend/sugarsync/sugarsync.go b/backend/sugarsync/sugarsync.go index c81042480..de553ba2a 100644 --- a/backend/sugarsync/sugarsync.go +++ b/backend/sugarsync/sugarsync.go @@ -395,9 +395,7 @@ func parseExpiry(expiryString string) time.Time { } // NewFs constructs an Fs from the path, container:path -func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { - ctx := context.Background() - +func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { opt := new(Options) err := configstruct.Set(m, opt) if err != nil { diff --git a/backend/swift/swift.go b/backend/swift/swift.go index 68ffeefd3..01f11e467 100644 --- a/backend/swift/swift.go +++ b/backend/swift/swift.go @@ -432,7 +432,7 @@ func (f *Fs) setRoot(root string) { // // if noCheckContainer is set then the Fs won't check the container // exists before creating it. -func NewFsWithConnection(opt *Options, name, root string, c *swift.Connection, noCheckContainer bool) (fs.Fs, error) { +func NewFsWithConnection(ctx context.Context, opt *Options, name, root string, c *swift.Connection, noCheckContainer bool) (fs.Fs, error) { f := &Fs{ name: name, opt: *opt, @@ -473,7 +473,7 @@ func NewFsWithConnection(opt *Options, name, root string, c *swift.Connection, n } // NewFs constructs an Fs from the path, container:path -func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { +func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { // Parse config into Options struct opt := new(Options) err := configstruct.Set(m, opt) @@ -489,7 +489,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { if err != nil { return nil, err } - return NewFsWithConnection(opt, name, root, c, false) + return NewFsWithConnection(ctx, opt, name, root, c, false) } // Return an Object from a path diff --git a/backend/tardigrade/fs.go b/backend/tardigrade/fs.go index caf203d39..11d36189f 100644 --- a/backend/tardigrade/fs.go +++ b/backend/tardigrade/fs.go @@ -165,9 +165,7 @@ var ( ) // NewFs creates a filesystem backed by Tardigrade. -func NewFs(name, root string, m configmap.Mapper) (_ fs.Fs, err error) { - ctx := context.Background() - +func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (_ fs.Fs, err error) { // Setup filesystem and connection to Tardigrade root = norm.NFC.String(root) root = strings.Trim(root, "/") diff --git a/backend/union/union.go b/backend/union/union.go index 02174734a..9dd501260 100644 --- a/backend/union/union.go +++ b/backend/union/union.go @@ -757,7 +757,7 @@ func (f *Fs) mergeDirEntries(entriesList [][]upstream.Entry) (fs.DirEntries, err // NewFs constructs an Fs from the path. // // The returned Fs is the actual Fs, referenced by remote in the config -func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { +func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { // Parse config into Options struct opt := new(Options) err := configstruct.Set(m, opt) @@ -787,7 +787,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { errs := Errors(make([]error, len(opt.Upstreams))) multithread(len(opt.Upstreams), func(i int) { u := opt.Upstreams[i] - upstreams[i], errs[i] = upstream.New(u, root, time.Duration(opt.CacheTime)*time.Second) + upstreams[i], errs[i] = upstream.New(ctx, u, root, time.Duration(opt.CacheTime)*time.Second) }) var usedUpstreams []*upstream.Fs var fserr error diff --git a/backend/union/upstream/upstream.go b/backend/union/upstream/upstream.go index 4360b0b25..1c2c0417f 100644 --- a/backend/union/upstream/upstream.go +++ b/backend/union/upstream/upstream.go @@ -61,7 +61,7 @@ type Entry interface { // New creates a new Fs based on the // string formatted `type:root_path(:ro/:nc)` -func New(remote, root string, cacheTime time.Duration) (*Fs, error) { +func New(ctx context.Context, remote, root string, cacheTime time.Duration) (*Fs, error) { _, configName, fsPath, err := fs.ParseRemote(remote) if err != nil { return nil, err @@ -86,13 +86,13 @@ func New(remote, root string, cacheTime time.Duration) (*Fs, error) { if configName != "local" { fsPath = configName + ":" + fsPath } - rFs, err := cache.Get(fsPath) + rFs, err := cache.Get(ctx, fsPath) if err != nil && err != fs.ErrorIsFile { return nil, err } f.RootFs = rFs rootString := path.Join(fsPath, filepath.ToSlash(root)) - myFs, err := cache.Get(rootString) + myFs, err := cache.Get(ctx, rootString) if err != nil && err != fs.ErrorIsFile { return nil, err } diff --git a/backend/webdav/webdav.go b/backend/webdav/webdav.go index bbf8bc729..c65625e64 100644 --- a/backend/webdav/webdav.go +++ b/backend/webdav/webdav.go @@ -299,8 +299,7 @@ func (o *Object) filePath() string { } // NewFs constructs an Fs from the path, container:path -func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { - ctx := context.Background() +func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { // Parse config into Options struct opt := new(Options) err := configstruct.Set(m, opt) diff --git a/backend/yandex/yandex.go b/backend/yandex/yandex.go index e5eba1a83..52fc25a96 100644 --- a/backend/yandex/yandex.go +++ b/backend/yandex/yandex.go @@ -237,8 +237,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string, options *api. } // NewFs constructs an Fs from the path, container:path -func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { - ctx := context.TODO() +func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { // Parse config into Options struct opt := new(Options) err := configstruct.Set(m, opt) diff --git a/cmd/backend/backend.go b/cmd/backend/backend.go index e10b48163..190dd15fc 100644 --- a/cmd/backend/backend.go +++ b/cmd/backend/backend.go @@ -74,7 +74,7 @@ Note to run these commands on a running backend then see if err != nil { return err } - f, err := fsInfo.NewFs(configName, fsPath, config) + f, err := fsInfo.NewFs(context.Background(), configName, fsPath, config) if err != nil { return err } diff --git a/cmd/cmd.go b/cmd/cmd.go index 309536410..85a361a74 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -7,6 +7,7 @@ package cmd // would probably mean bringing all the flags in to here? Or define some flagsets in fs... import ( + "context" "fmt" "log" "math/rand" @@ -88,7 +89,7 @@ func NewFsFile(remote string) (fs.Fs, string) { err = fs.CountError(err) log.Fatalf("Failed to create file system for %q: %v", remote, err) } - f, err := cache.Get(remote) + f, err := cache.Get(context.Background(), remote) switch err { case fs.ErrorIsFile: cache.Pin(f) // pin indefinitely since it was on the CLI @@ -138,7 +139,7 @@ func NewFsSrc(args []string) fs.Fs { // // This must point to a directory func newFsDir(remote string) fs.Fs { - f, err := cache.Get(remote) + f, err := cache.Get(context.Background(), remote) if err != nil { err = fs.CountError(err) log.Fatalf("Failed to create file system for %q: %v", remote, err) @@ -192,7 +193,7 @@ func NewFsSrcDstFiles(args []string) (fsrc fs.Fs, srcFileName string, fdst fs.Fs log.Fatalf("%q is a directory", args[1]) } } - fdst, err := cache.Get(dstRemote) + fdst, err := cache.Get(context.Background(), dstRemote) switch err { case fs.ErrorIsFile: _ = fs.CountError(err) @@ -400,7 +401,7 @@ func initConfig() { } // Start the remote control server if configured - _, err = rcserver.Start(&rcflags.Opt) + _, err = rcserver.Start(context.Background(), &rcflags.Opt) if err != nil { log.Fatalf("Failed to start remote control: %v", err) } diff --git a/cmd/lsf/lsf_test.go b/cmd/lsf/lsf_test.go index 1b2ef0d97..72c0853b0 100644 --- a/cmd/lsf/lsf_test.go +++ b/cmd/lsf/lsf_test.go @@ -17,7 +17,7 @@ func TestDefaultLsf(t *testing.T) { fstest.Initialise() buf := new(bytes.Buffer) - f, err := fs.NewFs("testfiles") + f, err := fs.NewFs(context.Background(), "testfiles") require.NoError(t, err) err = Lsf(context.Background(), f, buf) @@ -33,7 +33,7 @@ func TestRecurseFlag(t *testing.T) { fstest.Initialise() buf := new(bytes.Buffer) - f, err := fs.NewFs("testfiles") + f, err := fs.NewFs(context.Background(), "testfiles") require.NoError(t, err) recurse = true @@ -54,7 +54,7 @@ func TestDirSlashFlag(t *testing.T) { fstest.Initialise() buf := new(bytes.Buffer) - f, err := fs.NewFs("testfiles") + f, err := fs.NewFs(context.Background(), "testfiles") require.NoError(t, err) dirSlash = true @@ -80,7 +80,7 @@ subdir func TestFormat(t *testing.T) { fstest.Initialise() - f, err := fs.NewFs("testfiles") + f, err := fs.NewFs(context.Background(), "testfiles") require.NoError(t, err) buf := new(bytes.Buffer) @@ -160,7 +160,7 @@ file3 func TestSeparator(t *testing.T) { fstest.Initialise() - f, err := fs.NewFs("testfiles") + f, err := fs.NewFs(context.Background(), "testfiles") require.NoError(t, err) format = "ps" @@ -188,7 +188,7 @@ subdir__SEP__-1 func TestWholeLsf(t *testing.T) { fstest.Initialise() - f, err := fs.NewFs("testfiles") + f, err := fs.NewFs(context.Background(), "testfiles") require.NoError(t, err) format = "pst" separator = "_+_" diff --git a/cmd/mountlib/rc.go b/cmd/mountlib/rc.go index c492aad0a..728c94b73 100644 --- a/cmd/mountlib/rc.go +++ b/cmd/mountlib/rc.go @@ -76,7 +76,7 @@ The vfsOpt are as described in options/get and can be seen in the the } // mountRc allows the mount command to be run from rc -func mountRc(_ context.Context, in rc.Params) (out rc.Params, err error) { +func mountRc(ctx context.Context, in rc.Params) (out rc.Params, err error) { mountPoint, err := in.GetString("mountPoint") if err != nil { return nil, err @@ -110,7 +110,7 @@ func mountRc(_ context.Context, in rc.Params) (out rc.Params, err error) { } // Get Fs.fs to be mounted from fs parameter in the params - fdst, err := rc.GetFs(in) + fdst, err := rc.GetFs(ctx, in) if err != nil { return nil, err } diff --git a/cmd/rcd/rcd.go b/cmd/rcd/rcd.go index 240b6a272..c12efd9e9 100644 --- a/cmd/rcd/rcd.go +++ b/cmd/rcd/rcd.go @@ -1,6 +1,7 @@ package rcd import ( + "context" "log" "github.com/rclone/rclone/cmd" @@ -39,7 +40,7 @@ See the [rc documentation](/rc/) for more info on the rc flags. rcflags.Opt.Files = args[0] } - s, err := rcserver.Start(&rcflags.Opt) + s, err := rcserver.Start(context.Background(), &rcflags.Opt) if err != nil { log.Fatalf("Failed to start remote control: %v", err) } diff --git a/cmd/serve/dlna/dlna_test.go b/cmd/serve/dlna/dlna_test.go index 540a59b44..193a76684 100644 --- a/cmd/serve/dlna/dlna_test.go +++ b/cmd/serve/dlna/dlna_test.go @@ -43,7 +43,7 @@ func startServer(t *testing.T, f fs.Fs) { func TestInit(t *testing.T) { config.LoadConfig() - f, err := fs.NewFs("testdata/files") + f, err := fs.NewFs(context.Background(), "testdata/files") l, _ := f.List(context.Background(), "") fmt.Println(l) require.NoError(t, err) diff --git a/cmd/serve/ftp/ftp.go b/cmd/serve/ftp/ftp.go index 22cee3ef6..195b0ef76 100644 --- a/cmd/serve/ftp/ftp.go +++ b/cmd/serve/ftp/ftp.go @@ -5,6 +5,7 @@ package ftp import ( + "context" "fmt" "io" "net" @@ -101,7 +102,7 @@ You can set a single username and password with the --user and --pass flags. cmd.CheckArgs(0, 0, command, args) } cmd.Run(false, false, command, func() error { - s, err := newServer(f, &Opt) + s, err := newServer(context.Background(), f, &Opt) if err != nil { return err } @@ -114,13 +115,14 @@ You can set a single username and password with the --user and --pass flags. type server struct { f fs.Fs srv *ftp.Server + ctx context.Context // for global config opt Options vfs *vfs.VFS proxy *proxy.Proxy } // Make a new FTP to serve the remote -func newServer(f fs.Fs, opt *Options) (*server, error) { +func newServer(ctx context.Context, f fs.Fs, opt *Options) (*server, error) { host, port, err := net.SplitHostPort(opt.ListenAddr) if err != nil { return nil, errors.New("Failed to parse host:port") @@ -132,10 +134,11 @@ func newServer(f fs.Fs, opt *Options) (*server, error) { s := &server{ f: f, + ctx: ctx, opt: *opt, } if proxyflags.Opt.AuthProxy != "" { - s.proxy = proxy.New(&proxyflags.Opt) + s.proxy = proxy.New(ctx, &proxyflags.Opt) } else { s.vfs = vfs.New(f, &vfsflags.Opt) } diff --git a/cmd/serve/ftp/ftp_test.go b/cmd/serve/ftp/ftp_test.go index ee3e65ec6..186b0fb34 100644 --- a/cmd/serve/ftp/ftp_test.go +++ b/cmd/serve/ftp/ftp_test.go @@ -8,6 +8,7 @@ package ftp import ( + "context" "testing" _ "github.com/rclone/rclone/backend/local" @@ -38,7 +39,7 @@ func TestFTP(t *testing.T) { opt.BasicUser = testUSER opt.BasicPass = testPASS - w, err := newServer(f, &opt) + w, err := newServer(context.Background(), f, &opt) assert.NoError(t, err) quit := make(chan struct{}) diff --git a/cmd/serve/http/http_test.go b/cmd/serve/http/http_test.go index 269f56237..b4d234265 100644 --- a/cmd/serve/http/http_test.go +++ b/cmd/serve/http/http_test.go @@ -70,7 +70,7 @@ func TestInit(t *testing.T) { require.NoError(t, filter.Active.AddRule("- hidden/**")) // Create a test Fs - f, err := fs.NewFs("testdata/files") + f, err := fs.NewFs(context.Background(), "testdata/files") require.NoError(t, err) // set date of datedObject to expectedTime diff --git a/cmd/serve/proxy/proxy.go b/cmd/serve/proxy/proxy.go index ff5730145..e55233fef 100644 --- a/cmd/serve/proxy/proxy.go +++ b/cmd/serve/proxy/proxy.go @@ -3,6 +3,7 @@ package proxy import ( "bytes" + "context" "crypto/sha256" "crypto/subtle" "encoding/json" @@ -118,6 +119,7 @@ var DefaultOpt = Options{ type Proxy struct { cmdLine []string // broken down command line vfsCache *libcache.Cache + ctx context.Context // for global config Opt Options } @@ -128,8 +130,9 @@ type cacheEntry struct { } // New creates a new proxy with the Options passed in -func New(opt *Options) *Proxy { +func New(ctx context.Context, opt *Options) *Proxy { return &Proxy{ + ctx: ctx, Opt: *opt, cmdLine: strings.Fields(opt.AuthProxy), vfsCache: libcache.New(), @@ -220,7 +223,7 @@ func (p *Proxy) call(user, auth string, isPublicKey bool) (value interface{}, er // Look for fs in the VFS cache value, err = p.vfsCache.Get(user, func(key string) (value interface{}, ok bool, err error) { // Create the Fs from the cache - f, err := cache.GetFn(fsString, func(fsString string) (fs.Fs, error) { + f, err := cache.GetFn(p.ctx, fsString, func(ctx context.Context, fsString string) (fs.Fs, error) { // Update the config with the default values for i := range fsInfo.Options { o := &fsInfo.Options[i] @@ -228,7 +231,7 @@ func (p *Proxy) call(user, auth string, isPublicKey bool) (value interface{}, er config.Set(o.Name, o.String()) } } - return fsInfo.NewFs(name, root, config) + return fsInfo.NewFs(ctx, name, root, config) }) if err != nil { return nil, false, err diff --git a/cmd/serve/proxy/proxy_test.go b/cmd/serve/proxy/proxy_test.go index 5e52e4b11..1488e7406 100644 --- a/cmd/serve/proxy/proxy_test.go +++ b/cmd/serve/proxy/proxy_test.go @@ -1,6 +1,7 @@ package proxy import ( + "context" "crypto/rand" "crypto/rsa" "crypto/sha256" @@ -21,7 +22,7 @@ func TestRun(t *testing.T) { opt := DefaultOpt cmd := "go run proxy_code.go" opt.AuthProxy = cmd - p := New(&opt) + p := New(context.Background(), &opt) t.Run("Normal", func(t *testing.T) { config, err := p.run(map[string]string{ diff --git a/cmd/serve/sftp/server.go b/cmd/serve/sftp/server.go index 2899f2f55..e76d4652e 100644 --- a/cmd/serve/sftp/server.go +++ b/cmd/serve/sftp/server.go @@ -4,6 +4,7 @@ package sftp import ( "bytes" + "context" "crypto/rand" "crypto/rsa" "crypto/subtle" @@ -33,20 +34,22 @@ type server struct { f fs.Fs opt Options vfs *vfs.VFS + ctx context.Context // for global config config *ssh.ServerConfig listener net.Listener waitChan chan struct{} // for waiting on the listener to close proxy *proxy.Proxy } -func newServer(f fs.Fs, opt *Options) *server { +func newServer(ctx context.Context, f fs.Fs, opt *Options) *server { s := &server{ f: f, + ctx: ctx, opt: *opt, waitChan: make(chan struct{}), } if proxyflags.Opt.AuthProxy != "" { - s.proxy = proxy.New(&proxyflags.Opt) + s.proxy = proxy.New(ctx, &proxyflags.Opt) } else { s.vfs = vfs.New(f, &vfsflags.Opt) } diff --git a/cmd/serve/sftp/sftp.go b/cmd/serve/sftp/sftp.go index 8395de48c..13ddef6bf 100644 --- a/cmd/serve/sftp/sftp.go +++ b/cmd/serve/sftp/sftp.go @@ -5,6 +5,8 @@ package sftp import ( + "context" + "github.com/rclone/rclone/cmd" "github.com/rclone/rclone/cmd/serve/proxy" "github.com/rclone/rclone/cmd/serve/proxy/proxyflags" @@ -98,7 +100,7 @@ sftp backend, but it may not be with other SFTP clients. cmd.CheckArgs(0, 0, command, args) } cmd.Run(false, true, command, func() error { - s := newServer(f, &Opt) + s := newServer(context.Background(), f, &Opt) err := s.Serve() if err != nil { return err diff --git a/cmd/serve/sftp/sftp_test.go b/cmd/serve/sftp/sftp_test.go index baa81c196..19ff69f41 100644 --- a/cmd/serve/sftp/sftp_test.go +++ b/cmd/serve/sftp/sftp_test.go @@ -8,6 +8,7 @@ package sftp import ( + "context" "strings" "testing" @@ -44,7 +45,7 @@ func TestSftp(t *testing.T) { opt.User = testUser opt.Pass = testPass - w := newServer(f, &opt) + w := newServer(context.Background(), f, &opt) require.NoError(t, w.serve()) // Read the host and port we started on diff --git a/cmd/serve/webdav/webdav.go b/cmd/serve/webdav/webdav.go index 37cc34860..049f1acf6 100644 --- a/cmd/serve/webdav/webdav.go +++ b/cmd/serve/webdav/webdav.go @@ -84,7 +84,7 @@ Use "rclone hashsum" to see the full list. fs.Debugf(f, "Using hash %v for ETag", hashType) } cmd.Run(false, false, command, func() error { - s := newWebDAV(f, &httpflags.Opt) + s := newWebDAV(context.Background(), f, &httpflags.Opt) err := s.serve() if err != nil { return err @@ -114,18 +114,20 @@ type WebDAV struct { _vfs *vfs.VFS // don't use directly, use getVFS webdavhandler *webdav.Handler proxy *proxy.Proxy + ctx context.Context // for global config } // check interface var _ webdav.FileSystem = (*WebDAV)(nil) // Make a new WebDAV to serve the remote -func newWebDAV(f fs.Fs, opt *httplib.Options) *WebDAV { +func newWebDAV(ctx context.Context, f fs.Fs, opt *httplib.Options) *WebDAV { w := &WebDAV{ - f: f, + f: f, + ctx: ctx, } if proxyflags.Opt.AuthProxy != "" { - w.proxy = proxy.New(&proxyflags.Opt) + w.proxy = proxy.New(ctx, &proxyflags.Opt) // override auth copyOpt := *opt copyOpt.Auth = w.auth diff --git a/cmd/serve/webdav/webdav_test.go b/cmd/serve/webdav/webdav_test.go index 8e461873a..32c50abbd 100644 --- a/cmd/serve/webdav/webdav_test.go +++ b/cmd/serve/webdav/webdav_test.go @@ -8,6 +8,7 @@ package webdav import ( + "context" "flag" "io/ioutil" "net/http" @@ -56,7 +57,7 @@ func TestWebDav(t *testing.T) { hashType = hash.MD5 // Start the server - w := newWebDAV(f, &opt) + w := newWebDAV(context.Background(), f, &opt) assert.NoError(t, w.serve()) // Config for the backend we'll use to connect to the server @@ -91,7 +92,7 @@ func TestHTTPFunction(t *testing.T) { require.NoError(t, filter.Active.AddRule("- hidden/**")) // Uses the same test files as http tests but with different golden. - f, err := fs.NewFs("../http/testdata/files") + f, err := fs.NewFs(context.Background(), "../http/testdata/files") assert.NoError(t, err) opt := httplib.DefaultOpt @@ -99,7 +100,7 @@ func TestHTTPFunction(t *testing.T) { opt.Template = testTemplate // Start the server - w := newWebDAV(f, &opt) + w := newWebDAV(context.Background(), f, &opt) assert.NoError(t, w.serve()) defer func() { w.Close() diff --git a/cmd/tree/tree_test.go b/cmd/tree/tree_test.go index 5e1119fbf..7d0f3cbe6 100644 --- a/cmd/tree/tree_test.go +++ b/cmd/tree/tree_test.go @@ -2,6 +2,7 @@ package tree import ( "bytes" + "context" "testing" "github.com/a8m/tree" @@ -17,7 +18,7 @@ func TestTree(t *testing.T) { buf := new(bytes.Buffer) - f, err := fs.NewFs("testfiles") + f, err := fs.NewFs(context.Background(), "testfiles") require.NoError(t, err) err = Tree(f, buf, new(tree.Options)) require.NoError(t, err) diff --git a/fs/cache/cache.go b/fs/cache/cache.go index 8257ea0dc..106c0bbc5 100644 --- a/fs/cache/cache.go +++ b/fs/cache/cache.go @@ -2,6 +2,7 @@ package cache import ( + "context" "runtime" "sync" @@ -40,11 +41,11 @@ func addMapping(fsString, canonicalName string) { // GetFn gets an fs.Fs named fsString either from the cache or creates // it afresh with the create function -func GetFn(fsString string, create func(fsString string) (fs.Fs, error)) (f fs.Fs, err error) { +func GetFn(ctx context.Context, fsString string, create func(ctx context.Context, fsString string) (fs.Fs, error)) (f fs.Fs, err error) { fsString = Canonicalize(fsString) created := false value, err := c.Get(fsString, func(fsString string) (f interface{}, ok bool, err error) { - f, err = create(fsString) + f, err = create(ctx, fsString) ok = err == nil || err == fs.ErrorIsFile created = ok return f, ok, err @@ -99,8 +100,8 @@ func Unpin(f fs.Fs) { } // Get gets an fs.Fs named fsString either from the cache or creates it afresh -func Get(fsString string) (f fs.Fs, err error) { - return GetFn(fsString, fs.NewFs) +func Get(ctx context.Context, fsString string) (f fs.Fs, err error) { + return GetFn(ctx, fsString, fs.NewFs) } // Put puts an fs.Fs named fsString into the cache diff --git a/fs/cache/cache_test.go b/fs/cache/cache_test.go index 56ad4231c..72e6ee019 100644 --- a/fs/cache/cache_test.go +++ b/fs/cache/cache_test.go @@ -1,6 +1,7 @@ package cache import ( + "context" "errors" "testing" @@ -15,9 +16,9 @@ var ( errSentinel = errors.New("an error") ) -func mockNewFs(t *testing.T) (func(), func(path string) (fs.Fs, error)) { +func mockNewFs(t *testing.T) (func(), func(ctx context.Context, path string) (fs.Fs, error)) { called = 0 - create := func(path string) (f fs.Fs, err error) { + create := func(ctx context.Context, path string) (f fs.Fs, err error) { assert.Equal(t, 0, called) called++ switch path { @@ -43,12 +44,12 @@ func TestGet(t *testing.T) { assert.Equal(t, 0, c.Entries()) - f, err := GetFn("mock:/", create) + f, err := GetFn(context.Background(), "mock:/", create) require.NoError(t, err) assert.Equal(t, 1, c.Entries()) - f2, err := GetFn("mock:/", create) + f2, err := GetFn(context.Background(), "mock:/", create) require.NoError(t, err) assert.Equal(t, f, f2) @@ -60,20 +61,20 @@ func TestGetFile(t *testing.T) { assert.Equal(t, 0, c.Entries()) - f, err := GetFn("mock:/file.txt", create) + f, err := GetFn(context.Background(), "mock:/file.txt", create) require.Equal(t, fs.ErrorIsFile, err) require.NotNil(t, f) assert.Equal(t, 2, c.Entries()) - f2, err := GetFn("mock:/file.txt", create) + f2, err := GetFn(context.Background(), "mock:/file.txt", create) require.Equal(t, fs.ErrorIsFile, err) require.NotNil(t, f2) assert.Equal(t, f, f2) // check parent is there too - f2, err = GetFn("mock:/", create) + f2, err = GetFn(context.Background(), "mock:/", create) require.Nil(t, err) require.NotNil(t, f2) @@ -86,20 +87,20 @@ func TestGetFile2(t *testing.T) { assert.Equal(t, 0, c.Entries()) - f, err := GetFn("mock:file.txt", create) + f, err := GetFn(context.Background(), "mock:file.txt", create) require.Equal(t, fs.ErrorIsFile, err) require.NotNil(t, f) assert.Equal(t, 2, c.Entries()) - f2, err := GetFn("mock:file.txt", create) + f2, err := GetFn(context.Background(), "mock:file.txt", create) require.Equal(t, fs.ErrorIsFile, err) require.NotNil(t, f2) assert.Equal(t, f, f2) // check parent is there too - f2, err = GetFn("mock:/", create) + f2, err = GetFn(context.Background(), "mock:/", create) require.Nil(t, err) require.NotNil(t, f2) @@ -112,7 +113,7 @@ func TestGetError(t *testing.T) { assert.Equal(t, 0, c.Entries()) - f, err := GetFn("mock:/error", create) + f, err := GetFn(context.Background(), "mock:/error", create) require.Equal(t, errSentinel, err) require.Equal(t, nil, f) @@ -131,7 +132,7 @@ func TestPut(t *testing.T) { assert.Equal(t, 1, c.Entries()) - fNew, err := GetFn("mock:/alien", create) + fNew, err := GetFn(context.Background(), "mock:/alien", create) require.NoError(t, err) require.Equal(t, f, fNew) @@ -141,7 +142,7 @@ func TestPut(t *testing.T) { Put("mock:/alien/", f) - fNew, err = GetFn("mock:/alien/", create) + fNew, err = GetFn(context.Background(), "mock:/alien/", create) require.NoError(t, err) require.Equal(t, f, fNew) @@ -159,7 +160,7 @@ func TestPin(t *testing.T) { Unpin(f) // Now test pinning an existing - f2, err := GetFn("mock:/", create) + f2, err := GetFn(context.Background(), "mock:/", create) require.NoError(t, err) Pin(f2) Unpin(f2) @@ -170,7 +171,7 @@ func TestClear(t *testing.T) { defer cleanup() // Create something - _, err := GetFn("mock:/", create) + _, err := GetFn(context.Background(), "mock:/", create) require.NoError(t, err) assert.Equal(t, 1, c.Entries()) diff --git a/fs/fs.go b/fs/fs.go index 6e416cdfe..b53c641b9 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -84,7 +84,7 @@ type RegInfo struct { // Create a new file system. If root refers to an existing // object, then it should return an Fs which which points to // the parent of that object and ErrorIsFile. - NewFs func(name string, root string, config configmap.Mapper) (Fs, error) `json:"-"` + NewFs func(ctx context.Context, name string, root string, config configmap.Mapper) (Fs, error) `json:"-"` // Function to call to help with config Config func(name string, config configmap.Mapper) `json:"-"` // Options for the Fs configuration @@ -1339,13 +1339,13 @@ func ConfigFs(path string) (fsInfo *RegInfo, configName, fsPath string, config * // // On Windows avoid single character remote names as they can be mixed // up with drive letters. -func NewFs(path string) (Fs, error) { +func NewFs(ctx context.Context, path string) (Fs, error) { Debugf(nil, "Creating backend with remote %q", path) fsInfo, configName, fsPath, config, err := ConfigFs(path) if err != nil { return nil, err } - return fsInfo.NewFs(configName, fsPath, config) + return fsInfo.NewFs(ctx, configName, fsPath, config) } // ConfigString returns a canonical version of the config string used @@ -1362,7 +1362,7 @@ func ConfigString(f Fs) string { // TemporaryLocalFs creates a local FS in the OS's temporary directory. // // No cleanup is performed, the caller must call Purge on the Fs themselves. -func TemporaryLocalFs() (Fs, error) { +func TemporaryLocalFs(ctx context.Context) (Fs, error) { path, err := ioutil.TempDir("", "rclone-spool") if err == nil { err = os.Remove(path) @@ -1371,7 +1371,7 @@ func TemporaryLocalFs() (Fs, error) { return nil, err } path = filepath.ToSlash(path) - return NewFs(path) + return NewFs(ctx, path) } // CheckClose is a utility function used to check the return from diff --git a/fs/operations/check_test.go b/fs/operations/check_test.go index ed9dc8dc8..8dda7b172 100644 --- a/fs/operations/check_test.go +++ b/fs/operations/check_test.go @@ -182,11 +182,11 @@ func TestCheck(t *testing.T) { } func TestCheckFsError(t *testing.T) { - dstFs, err := fs.NewFs("non-existent") + dstFs, err := fs.NewFs(context.Background(), "non-existent") if err != nil { t.Fatal(err) } - srcFs, err := fs.NewFs("non-existent") + srcFs, err := fs.NewFs(context.Background(), "non-existent") if err != nil { t.Fatal(err) } diff --git a/fs/operations/operations.go b/fs/operations/operations.go index 406e27a1c..784e50c86 100644 --- a/fs/operations/operations.go +++ b/fs/operations/operations.go @@ -1157,7 +1157,7 @@ func Rcat(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadCloser, canStream := fdst.Features().PutStream != nil if !canStream { fs.Debugf(fdst, "Target remote doesn't support streaming uploads, creating temporary local FS to spool file") - tmpLocalFs, err := fs.TemporaryLocalFs() + tmpLocalFs, err := fs.TemporaryLocalFs(ctx) if err != nil { return nil, errors.Wrap(err, "Failed to create temporary local FS to spool file") } @@ -1262,8 +1262,8 @@ func Rmdirs(ctx context.Context, f fs.Fs, dir string, leaveRoot bool) error { } // GetCompareDest sets up --compare-dest -func GetCompareDest() (CompareDest fs.Fs, err error) { - CompareDest, err = cache.Get(fs.Config.CompareDest) +func GetCompareDest(ctx context.Context) (CompareDest fs.Fs, err error) { + CompareDest, err = cache.Get(ctx, fs.Config.CompareDest) if err != nil { return nil, fserrors.FatalError(errors.Errorf("Failed to make fs for --compare-dest %q: %v", fs.Config.CompareDest, err)) } @@ -1298,8 +1298,8 @@ func compareDest(ctx context.Context, dst, src fs.Object, CompareDest fs.Fs) (No } // GetCopyDest sets up --copy-dest -func GetCopyDest(fdst fs.Fs) (CopyDest fs.Fs, err error) { - CopyDest, err = cache.Get(fs.Config.CopyDest) +func GetCopyDest(ctx context.Context, fdst fs.Fs) (CopyDest fs.Fs, err error) { + CopyDest, err = cache.Get(ctx, fs.Config.CopyDest) if err != nil { return nil, fserrors.FatalError(errors.Errorf("Failed to make fs for --copy-dest %q: %v", fs.Config.CopyDest, err)) } @@ -1530,9 +1530,9 @@ func CopyURLToWriter(ctx context.Context, url string, out io.Writer) (err error) } // BackupDir returns the correctly configured --backup-dir -func BackupDir(fdst fs.Fs, fsrc fs.Fs, srcFileName string) (backupDir fs.Fs, err error) { +func BackupDir(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, srcFileName string) (backupDir fs.Fs, err error) { if fs.Config.BackupDir != "" { - backupDir, err = cache.Get(fs.Config.BackupDir) + backupDir, err = cache.Get(ctx, fs.Config.BackupDir) if err != nil { return nil, fserrors.FatalError(errors.Errorf("Failed to make fs for --backup-dir %q: %v", fs.Config.BackupDir, err)) } @@ -1636,18 +1636,18 @@ func moveOrCopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName str var backupDir, copyDestDir fs.Fs if fs.Config.BackupDir != "" || fs.Config.Suffix != "" { - backupDir, err = BackupDir(fdst, fsrc, srcFileName) + backupDir, err = BackupDir(ctx, fdst, fsrc, srcFileName) if err != nil { return errors.Wrap(err, "creating Fs for --backup-dir failed") } } if fs.Config.CompareDest != "" { - copyDestDir, err = GetCompareDest() + copyDestDir, err = GetCompareDest(ctx) if err != nil { return err } } else if fs.Config.CopyDest != "" { - copyDestDir, err = GetCopyDest(fdst) + copyDestDir, err = GetCopyDest(ctx, fdst) if err != nil { return err } diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go index f35ed49cd..102fb3ec3 100644 --- a/fs/operations/operations_test.go +++ b/fs/operations/operations_test.go @@ -819,7 +819,7 @@ func TestCopyFileCompareDest(t *testing.T) { defer func() { fs.Config.CompareDest = "" }() - fdst, err := fs.NewFs(r.FremoteName + "/dst") + fdst, err := fs.NewFs(context.Background(), r.FremoteName+"/dst") require.NoError(t, err) // check empty dest, empty compare @@ -904,7 +904,7 @@ func TestCopyFileCopyDest(t *testing.T) { fs.Config.CopyDest = "" }() - fdst, err := fs.NewFs(r.FremoteName + "/dst") + fdst, err := fs.NewFs(context.Background(), r.FremoteName+"/dst") require.NoError(t, err) // check empty dest, empty copy diff --git a/fs/operations/rc.go b/fs/operations/rc.go index 755d46285..380608844 100644 --- a/fs/operations/rc.go +++ b/fs/operations/rc.go @@ -44,7 +44,7 @@ See the [lsjson command](/commands/rclone_lsjson/) for more information on the a // List the directory func rcList(ctx context.Context, in rc.Params) (out rc.Params, err error) { - f, remote, err := rc.GetFsAndRemote(in) + f, remote, err := rc.GetFsAndRemote(ctx, in) if err != nil { return nil, err } @@ -85,7 +85,7 @@ See the [about command](/commands/rclone_size/) command for more information on // About the remote func rcAbout(ctx context.Context, in rc.Params) (out rc.Params, err error) { - f, err := rc.GetFs(in) + f, err := rc.GetFs(ctx, in) if err != nil { return nil, err } @@ -131,11 +131,11 @@ func init() { // Copy a file func rcMoveOrCopyFile(ctx context.Context, in rc.Params, cp bool) (out rc.Params, err error) { - srcFs, srcRemote, err := rc.GetFsAndRemoteNamed(in, "srcFs", "srcRemote") + srcFs, srcRemote, err := rc.GetFsAndRemoteNamed(ctx, in, "srcFs", "srcRemote") if err != nil { return nil, err } - dstFs, dstRemote, err := rc.GetFsAndRemoteNamed(in, "dstFs", "dstRemote") + dstFs, dstRemote, err := rc.GetFsAndRemoteNamed(ctx, in, "dstFs", "dstRemote") if err != nil { return nil, err } @@ -190,9 +190,9 @@ func rcSingleCommand(ctx context.Context, in rc.Params, name string, noRemote bo remote string ) if noRemote { - f, err = rc.GetFs(in) + f, err = rc.GetFs(ctx, in) } else { - f, remote, err = rc.GetFsAndRemote(in) + f, remote, err = rc.GetFsAndRemote(ctx, in) } if err != nil { return nil, err @@ -291,7 +291,7 @@ See the [size command](/commands/rclone_size/) command for more information on t // Size a directory func rcSize(ctx context.Context, in rc.Params) (out rc.Params, err error) { - f, err := rc.GetFs(in) + f, err := rc.GetFs(ctx, in) if err != nil { return nil, err } @@ -329,7 +329,7 @@ See the [link command](/commands/rclone_link/) command for more information on t // Make a public link func rcPublicLink(ctx context.Context, in rc.Params) (out rc.Params, err error) { - f, remote, err := rc.GetFsAndRemote(in) + f, remote, err := rc.GetFsAndRemote(ctx, in) if err != nil { return nil, err } @@ -413,7 +413,7 @@ This command does not have a command line equivalent so use this instead: // Fsinfo the remote func rcFsInfo(ctx context.Context, in rc.Params) (out rc.Params, err error) { - f, err := rc.GetFs(in) + f, err := rc.GetFs(ctx, in) if err != nil { return nil, err } @@ -478,7 +478,7 @@ See the [backend](/commands/rclone_backend/) command for more information. // Make a public link func rcBackend(ctx context.Context, in rc.Params) (out rc.Params, err error) { - f, err := rc.GetFs(in) + f, err := rc.GetFs(ctx, in) if err != nil { return nil, err } diff --git a/fs/rc/cache.go b/fs/rc/cache.go index d821f5c09..958a1db3f 100644 --- a/fs/rc/cache.go +++ b/fs/rc/cache.go @@ -3,34 +3,36 @@ package rc import ( + "context" + "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/cache" ) // GetFsNamed gets an fs.Fs named fsName either from the cache or creates it afresh -func GetFsNamed(in Params, fsName string) (f fs.Fs, err error) { +func GetFsNamed(ctx context.Context, in Params, fsName string) (f fs.Fs, err error) { fsString, err := in.GetString(fsName) if err != nil { return nil, err } - return cache.Get(fsString) + return cache.Get(ctx, fsString) } // GetFs gets an fs.Fs named "fs" either from the cache or creates it afresh -func GetFs(in Params) (f fs.Fs, err error) { - return GetFsNamed(in, "fs") +func GetFs(ctx context.Context, in Params) (f fs.Fs, err error) { + return GetFsNamed(ctx, in, "fs") } // GetFsAndRemoteNamed gets the fsName parameter from in, makes a // remote or fetches it from the cache then gets the remoteName // parameter from in too. -func GetFsAndRemoteNamed(in Params, fsName, remoteName string) (f fs.Fs, remote string, err error) { +func GetFsAndRemoteNamed(ctx context.Context, in Params, fsName, remoteName string) (f fs.Fs, remote string, err error) { remote, err = in.GetString(remoteName) if err != nil { return } - f, err = GetFsNamed(in, fsName) + f, err = GetFsNamed(ctx, in, fsName) return } @@ -38,6 +40,6 @@ func GetFsAndRemoteNamed(in Params, fsName, remoteName string) (f fs.Fs, remote // GetFsAndRemote gets the `fs` parameter from in, makes a remote or // fetches it from the cache then gets the `remote` parameter from in // too. -func GetFsAndRemote(in Params) (f fs.Fs, remote string, err error) { - return GetFsAndRemoteNamed(in, "fs", "remote") +func GetFsAndRemote(ctx context.Context, in Params) (f fs.Fs, remote string, err error) { + return GetFsAndRemoteNamed(ctx, in, "fs", "remote") } diff --git a/fs/rc/cache_test.go b/fs/rc/cache_test.go index bad753417..19d935d36 100644 --- a/fs/rc/cache_test.go +++ b/fs/rc/cache_test.go @@ -1,6 +1,7 @@ package rc import ( + "context" "testing" "github.com/rclone/rclone/fs/cache" @@ -23,14 +24,14 @@ func TestGetFsNamed(t *testing.T) { in := Params{ "potato": "/", } - f, err := GetFsNamed(in, "potato") + f, err := GetFsNamed(context.Background(), in, "potato") require.NoError(t, err) assert.NotNil(t, f) in = Params{ "sausage": "/", } - f, err = GetFsNamed(in, "potato") + f, err = GetFsNamed(context.Background(), in, "potato") require.Error(t, err) assert.Nil(t, f) } @@ -41,7 +42,7 @@ func TestGetFs(t *testing.T) { in := Params{ "fs": "/", } - f, err := GetFs(in) + f, err := GetFs(context.Background(), in) require.NoError(t, err) assert.NotNil(t, f) } @@ -53,16 +54,16 @@ func TestGetFsAndRemoteNamed(t *testing.T) { "fs": "/", "remote": "hello", } - f, remote, err := GetFsAndRemoteNamed(in, "fs", "remote") + f, remote, err := GetFsAndRemoteNamed(context.Background(), in, "fs", "remote") require.NoError(t, err) assert.NotNil(t, f) assert.Equal(t, "hello", remote) - f, _, err = GetFsAndRemoteNamed(in, "fsX", "remote") + f, _, err = GetFsAndRemoteNamed(context.Background(), in, "fsX", "remote") require.Error(t, err) assert.Nil(t, f) - f, _, err = GetFsAndRemoteNamed(in, "fs", "remoteX") + f, _, err = GetFsAndRemoteNamed(context.Background(), in, "fs", "remoteX") require.Error(t, err) assert.Nil(t, f) @@ -75,7 +76,7 @@ func TestGetFsAndRemote(t *testing.T) { "fs": "/", "remote": "hello", } - f, remote, err := GetFsAndRemote(in) + f, remote, err := GetFsAndRemote(context.Background(), in) require.NoError(t, err) assert.NotNil(t, f) assert.Equal(t, "hello", remote) diff --git a/fs/rc/rcserver/rcserver.go b/fs/rc/rcserver/rcserver.go index 6204c4810..4b904d7fa 100644 --- a/fs/rc/rcserver/rcserver.go +++ b/fs/rc/rcserver/rcserver.go @@ -2,6 +2,7 @@ package rcserver import ( + "context" "encoding/base64" "encoding/json" "flag" @@ -49,11 +50,11 @@ func init() { // Start the remote control server if configured // // If the server wasn't configured the *Server returned may be nil -func Start(opt *rc.Options) (*Server, error) { +func Start(ctx context.Context, opt *rc.Options) (*Server, error) { jobs.SetOpt(opt) // set the defaults for jobs if opt.Enabled { // Serve on the DefaultServeMux so can have global registrations appear - s := newServer(opt, http.DefaultServeMux) + s := newServer(ctx, opt, http.DefaultServeMux) return s, s.Serve() } return nil, nil @@ -62,12 +63,13 @@ func Start(opt *rc.Options) (*Server, error) { // Server contains everything to run the rc server type Server struct { *httplib.Server + ctx context.Context // for global config files http.Handler pluginsHandler http.Handler opt *rc.Options } -func newServer(opt *rc.Options, mux *http.ServeMux) *Server { +func newServer(ctx context.Context, opt *rc.Options, mux *http.ServeMux) *Server { fileHandler := http.Handler(nil) pluginsHandler := http.Handler(nil) // Add some more mime types which are often missing @@ -113,6 +115,7 @@ func newServer(opt *rc.Options, mux *http.ServeMux) *Server { s := &Server{ Server: httplib.NewServer(mux, &opt.HTTPOptions), + ctx: ctx, opt: opt, files: fileHandler, pluginsHandler: pluginsHandler, @@ -332,7 +335,7 @@ func (s *Server) serveRoot(w http.ResponseWriter, r *http.Request) { } func (s *Server) serveRemote(w http.ResponseWriter, r *http.Request, path string, fsName string) { - f, err := cache.Get(fsName) + f, err := cache.Get(s.ctx, fsName) if err != nil { writeError(path, nil, w, errors.Wrap(err, "failed to make Fs"), http.StatusInternalServerError) return diff --git a/fs/rc/rcserver/rcserver_test.go b/fs/rc/rcserver/rcserver_test.go index 3bcfcbc81..678157b5d 100644 --- a/fs/rc/rcserver/rcserver_test.go +++ b/fs/rc/rcserver/rcserver_test.go @@ -2,6 +2,7 @@ package rcserver import ( "bytes" + "context" "fmt" "io" "io/ioutil" @@ -37,7 +38,7 @@ func TestRcServer(t *testing.T) { opt.Serve = true opt.Files = testFs mux := http.NewServeMux() - rcServer := newServer(&opt, mux) + rcServer := newServer(context.Background(), &opt, mux) assert.NoError(t, rcServer.Serve()) defer func() { rcServer.Close() @@ -85,7 +86,7 @@ type testRun struct { func testServer(t *testing.T, tests []testRun, opt *rc.Options) { mux := http.NewServeMux() opt.HTTPOptions.Template = testTemplate - rcServer := newServer(opt, mux) + rcServer := newServer(context.Background(), opt, mux) for _, test := range tests { t.Run(test.Name, func(t *testing.T) { method := test.Method diff --git a/fs/sync/rc.go b/fs/sync/rc.go index 50c360d48..b00d2adb1 100644 --- a/fs/sync/rc.go +++ b/fs/sync/rc.go @@ -33,11 +33,11 @@ See the [` + name + ` command](/commands/rclone_` + name + `/) command for more // Sync/Copy/Move a file func rcSyncCopyMove(ctx context.Context, in rc.Params, name string) (out rc.Params, err error) { - srcFs, err := rc.GetFsNamed(in, "srcFs") + srcFs, err := rc.GetFsNamed(ctx, in, "srcFs") if err != nil { return nil, err } - dstFs, err := rc.GetFsNamed(in, "dstFs") + dstFs, err := rc.GetFsNamed(ctx, in, "dstFs") if err != nil { return nil, err } diff --git a/fs/sync/sync.go b/fs/sync/sync.go index ff33713b5..ee257d57e 100644 --- a/fs/sync/sync.go +++ b/fs/sync/sync.go @@ -201,20 +201,20 @@ func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete // Make Fs for --backup-dir if required if fs.Config.BackupDir != "" || fs.Config.Suffix != "" { var err error - s.backupDir, err = operations.BackupDir(fdst, fsrc, "") + s.backupDir, err = operations.BackupDir(ctx, fdst, fsrc, "") if err != nil { return nil, err } } if fs.Config.CompareDest != "" { var err error - s.compareCopyDest, err = operations.GetCompareDest() + s.compareCopyDest, err = operations.GetCompareDest(ctx) if err != nil { return nil, err } } else if fs.Config.CopyDest != "" { var err error - s.compareCopyDest, err = operations.GetCopyDest(fdst) + s.compareCopyDest, err = operations.GetCopyDest(ctx, fdst) if err != nil { return nil, err } diff --git a/fs/sync/sync_test.go b/fs/sync/sync_test.go index 1ac6f5494..4b20bfbd6 100644 --- a/fs/sync/sync_test.go +++ b/fs/sync/sync_test.go @@ -71,7 +71,7 @@ func TestCopyMissingDirectory(t *testing.T) { defer r.Finalise() r.Mkdir(context.Background(), r.Fremote) - nonExistingFs, err := fs.NewFs("/non-existing") + nonExistingFs, err := fs.NewFs(context.Background(), "/non-existing") if err != nil { t.Fatal(err) } @@ -1359,7 +1359,7 @@ func TestServerSideMoveOverlap(t *testing.T) { } subRemoteName := r.FremoteName + "/rclone-move-test" - FremoteMove, err := fs.NewFs(subRemoteName) + FremoteMove, err := fs.NewFs(context.Background(), subRemoteName) require.NoError(t, err) file1 := r.WriteObject(context.Background(), "potato2", "------------------------------------------------------------", t1) @@ -1384,7 +1384,7 @@ func TestSyncOverlap(t *testing.T) { defer r.Finalise() subRemoteName := r.FremoteName + "/rclone-sync-test" - FremoteSync, err := fs.NewFs(subRemoteName) + FremoteSync, err := fs.NewFs(context.Background(), subRemoteName) require.NoError(t, err) checkErr := func(err error) { @@ -1409,7 +1409,7 @@ func TestSyncCompareDest(t *testing.T) { fs.Config.CompareDest = "" }() - fdst, err := fs.NewFs(r.FremoteName + "/dst") + fdst, err := fs.NewFs(context.Background(), r.FremoteName+"/dst") require.NoError(t, err) // check empty dest, empty compare @@ -1500,7 +1500,7 @@ func TestSyncCopyDest(t *testing.T) { fs.Config.CopyDest = "" }() - fdst, err := fs.NewFs(r.FremoteName + "/dst") + fdst, err := fs.NewFs(context.Background(), r.FremoteName+"/dst") require.NoError(t, err) // check empty dest, empty copy @@ -1635,7 +1635,7 @@ func testSyncBackupDir(t *testing.T, backupDir string, suffix string, suffixKeep fstest.CheckItems(t, r.Fremote, file1, file2, file3) fstest.CheckItems(t, r.Flocal, file1a, file2a) - fdst, err := fs.NewFs(r.FremoteName + "/dst") + fdst, err := fs.NewFs(context.Background(), r.FremoteName+"/dst") require.NoError(t, err) accounting.GlobalStats().ResetCounters() @@ -1723,7 +1723,7 @@ func testSyncSuffix(t *testing.T, suffix string, suffixKeepExtension bool) { fstest.CheckItems(t, r.Fremote, file1, file2, file3) fstest.CheckItems(t, r.Flocal, file1a, file2a, file3a) - fdst, err := fs.NewFs(r.FremoteName + "/dst") + fdst, err := fs.NewFs(context.Background(), r.FremoteName+"/dst") require.NoError(t, err) accounting.GlobalStats().ResetCounters() diff --git a/fstest/fstest.go b/fstest/fstest.go index 8b30c3c96..00db45273 100644 --- a/fstest/fstest.go +++ b/fstest/fstest.go @@ -453,7 +453,7 @@ func RandomRemote() (fs.Fs, string, func(), error) { return nil, "", nil, err } - remote, err := fs.NewFs(remoteName) + remote, err := fs.NewFs(context.Background(), remoteName) if err != nil { return nil, "", nil, err } diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go index 66db1e1e1..8a506df88 100644 --- a/fstest/fstests/fstests.go +++ b/fstest/fstests/fstests.go @@ -373,7 +373,7 @@ func Run(t *testing.T, opt *Opt) { // remote - the result of fs.NewFs(TestRemote:subRemoteName) subRemoteName, subRemoteLeaf, err = fstest.RandomRemoteName(remoteName) require.NoError(t, err) - remote, err = fs.NewFs(subRemoteName) + remote, err = fs.NewFs(context.Background(), subRemoteName) if err == fs.ErrorNotFoundInConfigFile { t.Logf("Didn't find %q in config file - skipping tests", remoteName) return @@ -889,7 +889,7 @@ func Run(t *testing.T, opt *Opt) { // TestFsListDirRoot tests that DirList works in the root TestFsListDirRoot := func(t *testing.T) { skipIfNotOk(t) - rootRemote, err := fs.NewFs(remoteName) + rootRemote, err := fs.NewFs(context.Background(), remoteName) require.NoError(t, err) _, dirs, err := walk.GetAll(ctx, rootRemote, "", true, 1) require.NoError(t, err) @@ -1374,7 +1374,7 @@ func Run(t *testing.T, opt *Opt) { remoteName := subRemoteName + "/" + file2.Path file2Copy := file2 file2Copy.Path = "z.txt" - fileRemote, err := fs.NewFs(remoteName) + fileRemote, err := fs.NewFs(context.Background(), remoteName) require.NotNil(t, fileRemote) assert.Equal(t, fs.ErrorIsFile, err) @@ -1390,7 +1390,7 @@ func Run(t *testing.T, opt *Opt) { t.Run("FsIsFileNotFound", func(t *testing.T) { skipIfNotOk(t) remoteName := subRemoteName + "/not found.txt" - fileRemote, err := fs.NewFs(remoteName) + fileRemote, err := fs.NewFs(context.Background(), remoteName) require.NoError(t, err) fstest.CheckListing(t, fileRemote, []fstest.Item{}) }) @@ -1409,7 +1409,7 @@ func Run(t *testing.T, opt *Opt) { configName += ":" } t.Logf("Opening root remote %q path %q from %q", configName, configLeaf, subRemoteName) - rootRemote, err := fs.NewFs(configName) + rootRemote, err := fs.NewFs(context.Background(), configName) require.NoError(t, err) file1Root := file1 @@ -1851,7 +1851,7 @@ func Run(t *testing.T, opt *Opt) { // https://github.com/rclone/rclone/issues/3164. t.Run("FsRootCollapse", func(t *testing.T) { deepRemoteName := subRemoteName + "/deeper/nonexisting/directory" - deepRemote, err := fs.NewFs(deepRemoteName) + deepRemote, err := fs.NewFs(context.Background(), deepRemoteName) require.NoError(t, err) colonIndex := strings.IndexRune(deepRemoteName, ':') diff --git a/fstest/run.go b/fstest/run.go index 532fe3a84..3cd8ebc24 100644 --- a/fstest/run.go +++ b/fstest/run.go @@ -102,7 +102,7 @@ func newRun() *Run { r.Fatalf("Failed to create temp dir: %v", err) } r.LocalName = filepath.ToSlash(r.LocalName) - r.Flocal, err = fs.NewFs(r.LocalName) + r.Flocal, err = fs.NewFs(context.Background(), r.LocalName) if err != nil { r.Fatalf("Failed to make %q: %v", r.LocalName, err) } diff --git a/fstest/test_all/clean.go b/fstest/test_all/clean.go index 3eb40ca4d..7cda57093 100644 --- a/fstest/test_all/clean.go +++ b/fstest/test_all/clean.go @@ -20,7 +20,7 @@ var MatchTestRemote = regexp.MustCompile(`^rclone-test-[abcdefghijklmnopqrstuvwx // cleanFs runs a single clean fs for left over directories func cleanFs(ctx context.Context, remote string, cleanup bool) error { - f, err := fs.NewFs(remote) + f, err := fs.NewFs(context.Background(), remote) if err != nil { return err } @@ -46,7 +46,7 @@ func cleanFs(ctx context.Context, remote string, cleanup bool) error { return nil } log.Printf("Purging %s", fullPath) - dir, err := fs.NewFs(fullPath) + dir, err := fs.NewFs(context.Background(), fullPath) if err != nil { err = errors.Wrap(err, "NewFs failed") lastErr = err diff --git a/vfs/vfscache/cache.go b/vfs/vfscache/cache.go index 5a5dd8fa6..895c8b93a 100644 --- a/vfs/vfscache/cache.go +++ b/vfs/vfscache/cache.go @@ -86,11 +86,11 @@ func New(ctx context.Context, fremote fs.Fs, opt *vfscommon.Options, avFn AddVir metaRoot := file.UNCPath(filepath.Join(config.CacheDir, "vfsMeta", fremote.Name(), fRoot)) fs.Debugf(nil, "vfs cache: metadata root is %q", root) - fcache, err := fscache.Get(root) + fcache, err := fscache.Get(ctx, root) if err != nil { return nil, errors.Wrap(err, "failed to create cache remote") } - fcacheMeta, err := fscache.Get(root) + fcacheMeta, err := fscache.Get(ctx, root) if err != nil { return nil, errors.Wrap(err, "failed to create cache meta remote") }