alias,cache,chunker,crypt: make any created backends be cached to fix rc problems

Before this change, when the above backends created a new backend they
didn't put it into the backend cache.

This meant that rc commands acting on those backends did not work.

This was fixed by making sure the backends use the backend cache.

See: https://forum.rclone.org/t/rclone-rc-backend-command-not-working-as-expected/18834
This commit is contained in:
Nick Craig-Wood 2020-08-31 17:07:26 +01:00
parent 3affc2e066
commit 0d066bdf46
4 changed files with 26 additions and 28 deletions

View File

@ -5,6 +5,7 @@ import (
"strings" "strings"
"github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/cache"
"github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct" "github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/fspath" "github.com/rclone/rclone/fs/fspath"
@ -46,9 +47,5 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
if strings.HasPrefix(opt.Remote, name+":") { if strings.HasPrefix(opt.Remote, name+":") {
return nil, errors.New("can't point alias remote at itself - check the value of the remote setting") return nil, errors.New("can't point alias remote at itself - check the value of the remote setting")
} }
fsInfo, configName, fsPath, config, err := fs.ConfigFs(opt.Remote) return cache.Get(fspath.JoinRootPath(opt.Remote, root))
if err != nil {
return nil, err
}
return fsInfo.NewFs(configName, fspath.JoinRootPath(fsPath, root), config)
} }

View File

@ -361,15 +361,10 @@ func NewFs(name, rootPath string, m configmap.Mapper) (fs.Fs, error) {
return nil, errors.Wrapf(err, "failed to clean root path %q", rootPath) return nil, errors.Wrapf(err, "failed to clean root path %q", rootPath)
} }
wInfo, wName, wPath, wConfig, err := fs.ConfigFs(opt.Remote) remotePath := fspath.JoinRootPath(opt.Remote, rootPath)
if err != nil { wrappedFs, wrapErr := cache.Get(remotePath)
return nil, errors.Wrapf(err, "failed to parse remote %q to wrap", opt.Remote)
}
remotePath := fspath.JoinRootPath(wPath, rootPath)
wrappedFs, wrapErr := wInfo.NewFs(wName, remotePath, wConfig)
if wrapErr != nil && wrapErr != fs.ErrorIsFile { if wrapErr != nil && wrapErr != fs.ErrorIsFile {
return nil, errors.Wrapf(wrapErr, "failed to make remote %s:%s to wrap", wName, remotePath) return nil, errors.Wrapf(wrapErr, "failed to make remote %q to wrap", remotePath)
} }
var fsErr error var fsErr error
fs.Debugf(name, "wrapped %v:%v at root %v", wrappedFs.Name(), wrappedFs.Root(), rpath) fs.Debugf(name, "wrapped %v:%v at root %v", wrappedFs.Name(), wrappedFs.Root(), rpath)

View File

@ -24,6 +24,7 @@ import (
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/accounting" "github.com/rclone/rclone/fs/accounting"
"github.com/rclone/rclone/fs/cache"
"github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct" "github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/fspath" "github.com/rclone/rclone/fs/fspath"
@ -238,15 +239,18 @@ func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) {
return nil, errors.New("can't point remote at itself - check the value of the remote setting") return nil, errors.New("can't point remote at itself - check the value of the remote setting")
} }
baseInfo, baseName, basePath, baseConfig, err := fs.ConfigFs(remote) baseName, basePath, err := fspath.Parse(remote)
if err != nil { if err != nil {
return nil, errors.Wrapf(err, "failed to parse remote %q to wrap", remote) return nil, errors.Wrapf(err, "failed to parse remote %q to wrap", remote)
} }
if baseName != "" {
baseName += ":"
}
// Look for a file first // Look for a file first
remotePath := fspath.JoinRootPath(basePath, rpath) remotePath := fspath.JoinRootPath(basePath, rpath)
baseFs, err := baseInfo.NewFs(baseName, remotePath, baseConfig) baseFs, err := cache.Get(baseName + remotePath)
if err != fs.ErrorIsFile && err != nil { if err != fs.ErrorIsFile && err != nil {
return nil, errors.Wrapf(err, "failed to make remote %s:%q to wrap", baseName, remotePath) return nil, errors.Wrapf(err, "failed to make remote %q to wrap", baseName+remotePath)
} }
if !operations.CanServerSideMove(baseFs) { if !operations.CanServerSideMove(baseFs) {
return nil, errors.New("can't use chunker on a backend which doesn't support server side move or copy") return nil, errors.New("can't use chunker on a backend which doesn't support server side move or copy")
@ -271,7 +275,7 @@ func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) {
// (yet can't satisfy fstest.CheckListing, will ignore) // (yet can't satisfy fstest.CheckListing, will ignore)
if err == nil && !f.useMeta && strings.Contains(rpath, "/") { if err == nil && !f.useMeta && strings.Contains(rpath, "/") {
firstChunkPath := f.makeChunkName(remotePath, 0, "", "") firstChunkPath := f.makeChunkName(remotePath, 0, "", "")
_, testErr := baseInfo.NewFs(baseName, firstChunkPath, baseConfig) _, testErr := cache.Get(baseName + firstChunkPath)
if testErr == fs.ErrorIsFile { if testErr == fs.ErrorIsFile {
err = testErr err = testErr
} }

View File

@ -12,6 +12,7 @@ import (
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/accounting" "github.com/rclone/rclone/fs/accounting"
"github.com/rclone/rclone/fs/cache"
"github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct" "github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/config/obscure" "github.com/rclone/rclone/fs/config/obscure"
@ -158,24 +159,25 @@ func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) {
if strings.HasPrefix(remote, name+":") { if strings.HasPrefix(remote, name+":") {
return nil, errors.New("can't point crypt remote at itself - check the value of the remote setting") return nil, errors.New("can't point crypt remote at itself - check the value of the remote setting")
} }
wInfo, wName, wPath, wConfig, err := fs.ConfigFs(remote)
if err != nil {
return nil, errors.Wrapf(err, "failed to parse remote %q to wrap", remote)
}
// Make sure to remove trailing . reffering to the current dir // Make sure to remove trailing . reffering to the current dir
if path.Base(rpath) == "." { if path.Base(rpath) == "." {
rpath = strings.TrimSuffix(rpath, ".") rpath = strings.TrimSuffix(rpath, ".")
} }
// Look for a file first // Look for a file first
remotePath := fspath.JoinRootPath(wPath, cipher.EncryptFileName(rpath)) var wrappedFs fs.Fs
wrappedFs, err := wInfo.NewFs(wName, remotePath, wConfig) if rpath == "" {
// if that didn't produce a file, look for a directory wrappedFs, err = cache.Get(remote)
if err != fs.ErrorIsFile { } else {
remotePath = fspath.JoinRootPath(wPath, cipher.EncryptDirName(rpath)) remotePath := fspath.JoinRootPath(remote, cipher.EncryptFileName(rpath))
wrappedFs, err = wInfo.NewFs(wName, remotePath, wConfig) wrappedFs, err = cache.Get(remotePath)
// if that didn't produce a file, look for a directory
if err != fs.ErrorIsFile {
remotePath = fspath.JoinRootPath(remote, cipher.EncryptDirName(rpath))
wrappedFs, err = cache.Get(remotePath)
}
} }
if err != fs.ErrorIsFile && err != nil { if err != fs.ErrorIsFile && err != nil {
return nil, errors.Wrapf(err, "failed to make remote %s:%q to wrap", wName, remotePath) return nil, errors.Wrapf(err, "failed to make remote %q to wrap", remote)
} }
f := &Fs{ f := &Fs{
Fs: wrappedFs, Fs: wrappedFs,