cache: add the ability to specify a custom chunk path - fixes #1872

cache/cache.go

@@ -48,6 +48,7 @@ const (
 var (
 	// Flags
 	cacheDbPath             = fs.StringP("cache-db-path", "", filepath.Join(fs.CacheDir, "cache-backend"), "Directory to cache DB")
+	cacheChunkPath          = fs.StringP("cache-chunk-path", "", filepath.Join(fs.CacheDir, "cache-backend"), "Directory to cached chunk files")
 	cacheDbPurge            = fs.BoolP("cache-db-purge", "", false, "Purge the cache DB before")
 	cacheChunkSize          = fs.StringP("cache-chunk-size", "", DefCacheChunkSize, "The size of a chunk")
 	cacheTotalChunkSize     = fs.StringP("cache-total-chunk-size", "", DefCacheTotalChunkSize, "The total size which the chunks can take up from the disk")
@@ -316,17 +317,32 @@ func NewFs(name, rpath string) (fs.Fs, error) {
 	}
 
 	dbPath := *cacheDbPath
+	chunkPath := *cacheChunkPath
+	// if the dbPath is non default but the chunk path is default, we overwrite the last to follow the same one as dbPath
+	if dbPath != filepath.Join(fs.CacheDir, "cache-backend") &&
+		chunkPath == filepath.Join(fs.CacheDir, "cache-backend") {
+		chunkPath = dbPath
+	}
 	if filepath.Ext(dbPath) != "" {
 		dbPath = filepath.Dir(dbPath)
 	}
+	if filepath.Ext(chunkPath) != "" {
+		chunkPath = filepath.Dir(chunkPath)
+	}
 	err = os.MkdirAll(dbPath, os.ModePerm)
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to create cache directory %v", dbPath)
 	}
+	err = os.MkdirAll(chunkPath, os.ModePerm)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to create cache directory %v", chunkPath)
+	}
 
 	dbPath = filepath.Join(dbPath, name+".db")
-	fs.Infof(name, "Storage DB path: %v", dbPath)
-	f.cache, err = GetPersistent(dbPath, &Features{
+	chunkPath = filepath.Join(chunkPath, name)
+	fs.Infof(name, "Cache DB path: %v", dbPath)
+	fs.Infof(name, "Cache chunk path: %v", chunkPath)
+	f.cache, err = GetPersistent(dbPath, chunkPath, &Features{
 		PurgeDb: *cacheDbPurge,
 	})
 	if err != nil {

cache/storage_persistent.go

@@ -15,7 +15,6 @@ import (
 	"sync"
 
 	"io/ioutil"
-	"path/filepath"
 
 	bolt "github.com/coreos/bbolt"
 	"github.com/ncw/rclone/fs"
@@ -38,7 +37,7 @@ var boltMap = make(map[string]*Persistent)
 var boltMapMx sync.Mutex
 
 // GetPersistent returns a single instance for the specific store
-func GetPersistent(dbPath string, f *Features) (*Persistent, error) {
+func GetPersistent(dbPath, chunkPath string, f *Features) (*Persistent, error) {
 	// write lock to create one
 	boltMapMx.Lock()
 	defer boltMapMx.Unlock()
@@ -46,7 +45,7 @@ func GetPersistent(dbPath string, f *Features) (*Persistent, error) {
 		return b, nil
 	}
 
-	bb, err := newPersistent(dbPath, f)
+	bb, err := newPersistent(dbPath, chunkPath, f)
 	if err != nil {
 		return nil, err
 	}
@@ -72,12 +71,10 @@ type Persistent struct {
 }
 
 // newPersistent builds a new wrapper and connects to the bolt.DB file
-func newPersistent(dbPath string, f *Features) (*Persistent, error) {
-	dataPath := strings.TrimSuffix(dbPath, filepath.Ext(dbPath))
-
+func newPersistent(dbPath, chunkPath string, f *Features) (*Persistent, error) {
 	b := &Persistent{
 		dbPath:   dbPath,
-		dataPath: dataPath,
+		dataPath: chunkPath,
 		features: f,
 	}
 

docs/content/cache.md

@@ -177,13 +177,25 @@ Organizing the remotes in this order yelds better results:
 Here are the command line options specific to this cloud storage
 system.
 
+#### --cache-chunk-path=PATH ####
+
+Path to where partial file data (chunks) is stored locally. The remote
+name is appended to the final path.
+
+This config follows the `--cache-db-path`. If you specify a custom
+location for `--cache-db-path` and don't specify one for `--cache-chunk-path`
+then `--cache-chunk-path` will use the same path as `--cache-db-path`.
+
+**Default**: <rclone default cache path>/cache-backend/<remote name>
+**Example**: /.cache/cache-backend/test-cache
+
 #### --cache-db-path=PATH ####
 
-Path to where partial file data (chunks) and the file structure metadata 
-are stored locally.
+Path to where the file structure metadata (DB) is stored locally. The remote
+name is used as the DB file name.
 
-**Default**: <rclone default config path>/<remote name>
-**Example**: ~/.config/rclone/test-cache
+**Default**: <rclone default cache path>/cache-backend/<remote name>
+**Example**: /.cache/cache-backend/test-cache
 
 #### --cache-db-purge ####