vfs: add the vfs/refresh rc command

vfs/refresh will walk the directory tree for the given paths and
freshen the directory cache. It will use the fast-list capability
of the remote when enabled.
This commit is contained in:
Fabian Möller 2018-07-27 16:36:22 +02:00 committed by Nick Craig-Wood
parent 38381d3786
commit 782972088d
2 changed files with 129 additions and 2 deletions

View File

@ -10,6 +10,7 @@ import (
"github.com/ncw/rclone/fs" "github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fs/list" "github.com/ncw/rclone/fs/list"
"github.com/ncw/rclone/fs/walk"
"github.com/pkg/errors" "github.com/pkg/errors"
) )
@ -187,6 +188,21 @@ func (d *Dir) _readDir() error {
return err return err
} }
err = d._readDirFromEntries(entries, nil, time.Time{})
if err != nil {
return err
}
d.read = when
return nil
}
func (d *Dir) _readDirFromDirTree(dirTree walk.DirTree, when time.Time) error {
return d._readDirFromEntries(dirTree[d.path], dirTree, when)
}
func (d *Dir) _readDirFromEntries(entries fs.DirEntries, dirTree walk.DirTree, when time.Time) error {
var err error
// Cache the items by name // Cache the items by name
found := make(map[string]struct{}) found := make(map[string]struct{})
for _, entry := range entries { for _, entry := range entries {
@ -206,10 +222,23 @@ func (d *Dir) _readDir() error {
node = newFile(d, obj, name) node = newFile(d, obj, name)
} }
case fs.Directory: case fs.Directory:
dir := item
// Reuse old dir value if it exists // Reuse old dir value if it exists
if node == nil || !node.IsDir() { if node == nil || !node.IsDir() {
node = newDir(d.vfs, d.f, d, dir) node = newDir(d.vfs, d.f, d, item)
}
if dirTree != nil {
dir := node.(*Dir)
dir.mu.Lock()
err = dir._readDirFromDirTree(dirTree, when)
if err != nil {
dir.read = time.Time{}
} else {
dir.read = when
}
dir.mu.Unlock()
if err != nil {
return err
}
} }
default: default:
err = errors.Errorf("unknown type %T", item) err = errors.Errorf("unknown type %T", item)
@ -224,6 +253,24 @@ func (d *Dir) _readDir() error {
delete(d.items, name) delete(d.items, name)
} }
} }
return nil
}
func (d *Dir) readDirTree() error {
d.mu.Lock()
defer d.mu.Unlock()
when := time.Now()
d.read = time.Time{}
fs.Debugf(d.path, "Reading directory tree")
dt, err := walk.NewDirTree(d.f, d.path, false, -1)
if err != nil {
return err
}
err = d._readDirFromDirTree(dt, when)
if err != nil {
return err
}
fs.Debugf(d.path, "Reading directory tree done in %s", time.Since(when))
d.read = when d.read = when
return nil return nil
} }

View File

@ -59,6 +59,86 @@ starting with dir will forget that dir, eg
rclone rc vfs/forget file=hello file2=goodbye dir=home/junk rclone rc vfs/forget file=hello file2=goodbye dir=home/junk
`,
})
rc.Add(rc.Call{
Path: "vfs/refresh",
Fn: func(in rc.Params) (out rc.Params, err error) {
root, err := vfs.Root()
if err != nil {
return nil, err
}
getDir := func(path string) (*Dir, error) {
path = strings.Trim(path, "/")
segments := strings.Split(path, "/")
var node Node = root
for _, s := range segments {
if dir, ok := node.(*Dir); ok {
node, err = dir.stat(s)
if err != nil {
return nil, err
}
}
}
if dir, ok := node.(*Dir); ok {
return dir, nil
}
return nil, EINVAL
}
result := map[string]string{}
if len(in) == 0 {
err = root.readDirTree()
if err != nil {
result[""] = err.Error()
} else {
result[""] = "OK"
}
} else {
for k, v := range in {
path, ok := v.(string)
if !ok {
return out, errors.Errorf("value must be string %q=%v", k, v)
}
if strings.HasPrefix(k, "dir") {
dir, err := getDir(path)
if err != nil {
result[path] = err.Error()
} else {
err = dir.readDirTree()
if err != nil {
result[path] = err.Error()
} else {
result[path] = "OK"
}
}
} else {
return out, errors.Errorf("unknown key %q", k)
}
}
}
out = rc.Params{
"result": result,
}
return out, nil
},
Title: "Refresh the directory cache tree.",
Help: `
This reads the full directory tree for the paths and freshens the
directory cache.
If no paths are passed in then it will refresh the root directory.
rclone rc vfs/refresh
Otherwise pass directories in as dir=path. Any parameter key
starting with dir will refresh that directory, eg
rclone rc vfs/refresh dir=home/junk dir2=data/misc
This refresh will use --fast-list if enabled.
`, `,
}) })
} }