diff --git a/docs/content/docs.md b/docs/content/docs.md index ee0a6bd2a..9410dc658 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -787,6 +787,27 @@ triggering follow-on actions if data was copied, or skipping if not. NB: Enabling this option turns a usually non-fatal error into a potentially fatal one - please check and adjust your scripts accordingly! +### --fs-cache-expire-duration=TIME + +When using rclone via the API rclone caches created remotes for 5 +minutes by default in the "fs cache". This means that if you do +repeated actions on the same remote then rclone won't have to build it +again from scratch, which makes it more efficient. + +This flag sets the time that the remotes are cached for. If you set it +to `0` (or negative) then rclone won't cache the remotes at all. + +Note that if you use some flags, eg `--backup-dir` and if this is set +to `0` rclone may build two remotes (one for the source or destination +and one for the `--backup-dir` where it may have only built one +before. + +### --fs-cache-expire-interval=TIME + +This controls how often rclone checks for cached remotes to expire. +See the `--fs-cache-expire-duration` documentation above for more +info. The default is 60s, set to 0 to disable expiry. + ### --header ### Add an HTTP header for all transactions. The flag can be repeated to diff --git a/fs/cache/cache.go b/fs/cache/cache.go index 7e1bf2d32..77d99fbec 100644 --- a/fs/cache/cache.go +++ b/fs/cache/cache.go @@ -12,14 +12,26 @@ import ( ) var ( - c = cache.New() + once sync.Once // creation + c *cache.Cache mu sync.Mutex // mutex to protect remap remap = map[string]string{} // map user supplied names to canonical names ) +// Create the cache just once +func createOnFirstUse() { + once.Do(func() { + ci := fs.GetConfig(context.Background()) + c = cache.New() + c.SetExpireDuration(ci.FsCacheExpireDuration) + c.SetExpireInterval(ci.FsCacheExpireInterval) + }) +} + // Canonicalize looks up fsString in the mapping from user supplied // names to canonical names and return the canonical form func Canonicalize(fsString string) string { + createOnFirstUse() mu.Lock() canonicalName, ok := remap[fsString] mu.Unlock() @@ -43,6 +55,7 @@ func addMapping(fsString, canonicalName string) { // GetFn gets an fs.Fs named fsString either from the cache or creates // it afresh with the create function func GetFn(ctx context.Context, fsString string, create func(ctx context.Context, fsString string) (fs.Fs, error)) (f fs.Fs, err error) { + createOnFirstUse() fsString = Canonicalize(fsString) created := false value, err := c.Get(fsString, func(fsString string) (f interface{}, ok bool, err error) { @@ -80,6 +93,7 @@ func GetFn(ctx context.Context, fsString string, create func(ctx context.Context // Pin f into the cache until Unpin is called func Pin(f fs.Fs) { + createOnFirstUse() c.Pin(fs.ConfigString(f)) } @@ -97,6 +111,7 @@ func PinUntilFinalized(f fs.Fs, x interface{}) { // Unpin f from the cache func Unpin(f fs.Fs) { + createOnFirstUse() c.Pin(fs.ConfigString(f)) } @@ -127,6 +142,7 @@ func GetArr(ctx context.Context, fsStrings []string) (f []fs.Fs, err error) { // Put puts an fs.Fs named fsString into the cache func Put(fsString string, f fs.Fs) { + createOnFirstUse() canonicalName := fs.ConfigString(f) c.Put(canonicalName, f) addMapping(fsString, canonicalName) @@ -136,15 +152,18 @@ func Put(fsString string, f fs.Fs) { // // Returns number of entries deleted func ClearConfig(name string) (deleted int) { + createOnFirstUse() return c.DeletePrefix(name + ":") } // Clear removes everything from the cache func Clear() { + createOnFirstUse() c.Clear() } // Entries returns the number of entries in the cache func Entries() int { + createOnFirstUse() return c.Entries() } diff --git a/fs/cache/cache_test.go b/fs/cache/cache_test.go index c23739588..85d801ad9 100644 --- a/fs/cache/cache_test.go +++ b/fs/cache/cache_test.go @@ -33,7 +33,7 @@ func mockNewFs(t *testing.T) (func(), func(ctx context.Context, path string) (fs panic("unreachable") } cleanup := func() { - c.Clear() + Clear() } return cleanup, create } @@ -42,12 +42,12 @@ func TestGet(t *testing.T) { cleanup, create := mockNewFs(t) defer cleanup() - assert.Equal(t, 0, c.Entries()) + assert.Equal(t, 0, Entries()) f, err := GetFn(context.Background(), "mock:/", create) require.NoError(t, err) - assert.Equal(t, 1, c.Entries()) + assert.Equal(t, 1, Entries()) f2, err := GetFn(context.Background(), "mock:/", create) require.NoError(t, err) @@ -59,13 +59,13 @@ func TestGetFile(t *testing.T) { cleanup, create := mockNewFs(t) defer cleanup() - assert.Equal(t, 0, c.Entries()) + assert.Equal(t, 0, Entries()) f, err := GetFn(context.Background(), "mock:/file.txt", create) require.Equal(t, fs.ErrorIsFile, err) require.NotNil(t, f) - assert.Equal(t, 2, c.Entries()) + assert.Equal(t, 2, Entries()) f2, err := GetFn(context.Background(), "mock:/file.txt", create) require.Equal(t, fs.ErrorIsFile, err) @@ -85,13 +85,13 @@ func TestGetFile2(t *testing.T) { cleanup, create := mockNewFs(t) defer cleanup() - assert.Equal(t, 0, c.Entries()) + assert.Equal(t, 0, Entries()) f, err := GetFn(context.Background(), "mock:file.txt", create) require.Equal(t, fs.ErrorIsFile, err) require.NotNil(t, f) - assert.Equal(t, 2, c.Entries()) + assert.Equal(t, 2, Entries()) f2, err := GetFn(context.Background(), "mock:file.txt", create) require.Equal(t, fs.ErrorIsFile, err) @@ -111,13 +111,13 @@ func TestGetError(t *testing.T) { cleanup, create := mockNewFs(t) defer cleanup() - assert.Equal(t, 0, c.Entries()) + assert.Equal(t, 0, Entries()) f, err := GetFn(context.Background(), "mock:/error", create) require.Equal(t, errSentinel, err) require.Equal(t, nil, f) - assert.Equal(t, 0, c.Entries()) + assert.Equal(t, 0, Entries()) } func TestPut(t *testing.T) { @@ -126,17 +126,17 @@ func TestPut(t *testing.T) { f := mockfs.NewFs(context.Background(), "mock", "/alien") - assert.Equal(t, 0, c.Entries()) + assert.Equal(t, 0, Entries()) Put("mock:/alien", f) - assert.Equal(t, 1, c.Entries()) + assert.Equal(t, 1, Entries()) fNew, err := GetFn(context.Background(), "mock:/alien", create) require.NoError(t, err) require.Equal(t, f, fNew) - assert.Equal(t, 1, c.Entries()) + assert.Equal(t, 1, Entries()) // Check canonicalisation @@ -146,7 +146,7 @@ func TestPut(t *testing.T) { require.NoError(t, err) require.Equal(t, f, fNew) - assert.Equal(t, 1, c.Entries()) + assert.Equal(t, 1, Entries()) } @@ -170,7 +170,7 @@ func TestClearConfig(t *testing.T) { cleanup, create := mockNewFs(t) defer cleanup() - assert.Equal(t, 0, c.Entries()) + assert.Equal(t, 0, Entries()) _, err := GetFn(context.Background(), "mock:/file.txt", create) require.Equal(t, fs.ErrorIsFile, err) @@ -190,11 +190,11 @@ func TestClear(t *testing.T) { _, err := GetFn(context.Background(), "mock:/", create) require.NoError(t, err) - assert.Equal(t, 1, c.Entries()) + assert.Equal(t, 1, Entries()) Clear() - assert.Equal(t, 0, c.Entries()) + assert.Equal(t, 0, Entries()) } func TestEntries(t *testing.T) { diff --git a/fs/config.go b/fs/config.go index a33c4cc1f..0e3ce187d 100644 --- a/fs/config.go +++ b/fs/config.go @@ -123,6 +123,8 @@ type ConfigInfo struct { RefreshTimes bool NoConsole bool TrafficClass uint8 + FsCacheExpireDuration time.Duration + FsCacheExpireInterval time.Duration } // NewConfig creates a new config with everything set to the default @@ -160,6 +162,8 @@ func NewConfig() *ConfigInfo { c.MultiThreadStreams = 4 c.TrackRenamesStrategy = "hash" + c.FsCacheExpireDuration = 300 * time.Second + c.FsCacheExpireInterval = 60 * time.Second return c } diff --git a/fs/config/configflags/configflags.go b/fs/config/configflags/configflags.go index 5523f3f74..6ef83c055 100644 --- a/fs/config/configflags/configflags.go +++ b/fs/config/configflags/configflags.go @@ -128,6 +128,8 @@ func AddFlags(ci *fs.ConfigInfo, flagSet *pflag.FlagSet) { flags.BoolVarP(flagSet, &ci.RefreshTimes, "refresh-times", "", ci.RefreshTimes, "Refresh the modtime of remote files.") flags.BoolVarP(flagSet, &ci.NoConsole, "no-console", "", ci.NoConsole, "Hide console window. Supported on Windows only.") flags.StringVarP(flagSet, &dscp, "dscp", "", "", "Set DSCP value to connections. Can be value or names, eg. CS1, LE, DF, AF21.") + flags.DurationVarP(flagSet, &ci.FsCacheExpireDuration, "fs-cache-expire-duration", "", ci.FsCacheExpireDuration, "cache remotes for this long (0 to disable caching)") + flags.DurationVarP(flagSet, &ci.FsCacheExpireInterval, "fs-cache-expire-interval", "", ci.FsCacheExpireInterval, "interval to check for expired remotes") } // ParseHeaders converts the strings passed in via the header flags into HTTPOptions diff --git a/lib/cache/cache.go b/lib/cache/cache.go index 9fe406f3c..f9533c137 100644 --- a/lib/cache/cache.go +++ b/lib/cache/cache.go @@ -28,6 +28,30 @@ func New() *Cache { } } +// SetExpireDuration sets the interval at which things expire +// +// If it is less than or equal to 0 then things are never cached +func (c *Cache) SetExpireDuration(d time.Duration) *Cache { + c.expireDuration = d + return c +} + +// returns true if we aren't to cache anything +func (c *Cache) noCache() bool { + return c.expireDuration <= 0 +} + +// SetExpireInterval sets the interval at which the cache expiry runs +// +// Set to 0 or a -ve number to disable +func (c *Cache) SetExpireInterval(d time.Duration) *Cache { + if d <= 0 { + d = 100 * 365 * 24 * time.Hour + } + c.expireInterval = d + return c +} + // cacheEntry is stored in the cache type cacheEntry struct { value interface{} // cached item @@ -69,7 +93,9 @@ func (c *Cache) Get(key string, create CreateFunc) (value interface{}, err error err: err, } c.mu.Lock() - c.cache[key] = entry + if !c.noCache() { + c.cache[key] = entry + } } defer c.mu.Unlock() c.used(entry) @@ -100,6 +126,9 @@ func (c *Cache) Unpin(key string) { func (c *Cache) Put(key string, value interface{}) { c.mu.Lock() defer c.mu.Unlock() + if c.noCache() { + return + } entry := &cacheEntry{ value: value, key: key, diff --git a/lib/cache/cache_test.go b/lib/cache/cache_test.go index f5530cacc..7a2366963 100644 --- a/lib/cache/cache_test.go +++ b/lib/cache/cache_test.go @@ -100,7 +100,7 @@ func TestPut(t *testing.T) { func TestCacheExpire(t *testing.T) { c, create := setup(t) - c.expireInterval = time.Millisecond + c.SetExpireInterval(time.Millisecond) assert.Equal(t, false, c.expireRunning) _, err := c.Get("/", create) @@ -127,6 +127,31 @@ func TestCacheExpire(t *testing.T) { c.mu.Unlock() } +func TestCacheNoExpire(t *testing.T) { + c, create := setup(t) + + assert.False(t, c.noCache()) + + c.SetExpireDuration(0) + assert.Equal(t, false, c.expireRunning) + + assert.True(t, c.noCache()) + + f, err := c.Get("/", create) + require.NoError(t, err) + require.NotNil(t, f) + + c.mu.Lock() + assert.Equal(t, 0, len(c.cache)) + c.mu.Unlock() + + c.Put("/alien", "slime") + + c.mu.Lock() + assert.Equal(t, 0, len(c.cache)) + c.mu.Unlock() +} + func TestCachePin(t *testing.T) { c, create := setup(t)