From 15402e46c9c5a6dae051c42bcd8560dd72eef467 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 23 Jun 2020 15:18:58 +0100 Subject: [PATCH] vfs: Add recovered items on cache reload to directory listings Before this change, if we restarted an upload after a restart then the file would get uploaded but never added to the directory listings. This change makes sure we add virtual items to the directory cache when reloading the cache so that they show up properly. --- vfs/vfs.go | 12 +++++++++++- vfs/vfscache/cache.go | 21 ++++++++++++++++++++- vfs/vfscache/cache_test.go | 20 +++++++++++++++++++- vfs/vfscache/item.go | 10 ++++++++++ vfs/vfscache/item_test.go | 5 +++++ 5 files changed, 65 insertions(+), 3 deletions(-) diff --git a/vfs/vfs.go b/vfs/vfs.go index ad94ef4f8..58d725648 100644 --- a/vfs/vfs.go +++ b/vfs/vfs.go @@ -259,7 +259,7 @@ func (vfs *VFS) SetCacheMode(cacheMode vfscommon.CacheMode) { vfs.cache = nil if cacheMode > vfscommon.CacheModeOff { ctx, cancel := context.WithCancel(context.Background()) - cache, err := vfscache.New(ctx, vfs.f, &vfs.Opt) // FIXME pass on context or get from Opt? + cache, err := vfscache.New(ctx, vfs.f, &vfs.Opt, vfs.AddVirtual) // FIXME pass on context or get from Opt? if err != nil { fs.Errorf(nil, "Failed to create vfs cache - disabling: %v", err) vfs.Opt.CacheMode = vfscommon.CacheModeOff @@ -652,3 +652,13 @@ func (vfs *VFS) ReadFile(filename string) (b []byte, err error) { defer fs.CheckClose(f, &err) return ioutil.ReadAll(f) } + +// AddVirtual adds the object (file or dir) to the directory cache +func (vfs *VFS) AddVirtual(remote string, size int64, isDir bool) error { + dir, leaf, err := vfs.StatParent(remote) + if err != nil { + return err + } + dir.AddVirtual(leaf, size, false) + return nil +} diff --git a/vfs/vfscache/cache.go b/vfs/vfscache/cache.go index a705a182d..e6f7454a6 100644 --- a/vfs/vfscache/cache.go +++ b/vfs/vfscache/cache.go @@ -45,17 +45,26 @@ type Cache struct { hashType hash.Type // hash to use locally and remotely hashOption *fs.HashesOption // corresponding OpenOption writeback *writeback.WriteBack // holds Items for writeback + avFn AddVirtualFn // if set, can be called to add dir entries mu sync.Mutex // protects the following variables item map[string]*Item // files/directories in the cache used int64 // total size of files in the cache } +// AddVirtualFn if registered by the WithAddVirtual method, can be +// called to register the object or directory at remote as a virtual +// entry in directory listings. +// +// This is used when reloading the Cache and uploading items need to +// go into the directory tree. +type AddVirtualFn func(remote string, size int64, isDir bool) error + // New creates a new cache heirachy for fremote // // This starts background goroutines which can be cancelled with the // context passed in. -func New(ctx context.Context, fremote fs.Fs, opt *vfscommon.Options) (*Cache, error) { +func New(ctx context.Context, fremote fs.Fs, opt *vfscommon.Options, avFn AddVirtualFn) (*Cache, error) { fRoot := filepath.FromSlash(fremote.Root()) if runtime.GOOS == "windows" { if strings.HasPrefix(fRoot, `\\?`) { @@ -90,6 +99,7 @@ func New(ctx context.Context, fremote fs.Fs, opt *vfscommon.Options) (*Cache, er hashType: hashType, hashOption: hashOption, writeback: writeback.New(ctx, opt), + avFn: avFn, } // Make sure cache directories exist @@ -567,3 +577,12 @@ func (c *Cache) Dump() string { out.WriteString("}\n") return out.String() } + +// AddVirtual adds a virtual directory entry by calling the addVirtual +// callback if one has been registered. +func (c *Cache) AddVirtual(remote string, size int64, isDir bool) error { + if c.avFn == nil { + return errors.New("no AddVirtual function registered") + } + return c.avFn(remote, size, isDir) +} diff --git a/vfs/vfscache/cache_test.go b/vfs/vfscache/cache_test.go index 8a728b9db..4418b7993 100644 --- a/vfs/vfscache/cache_test.go +++ b/vfs/vfscache/cache_test.go @@ -51,12 +51,30 @@ func assertPathExist(t *testing.T, path string) os.FileInfo { return fi } +type avInfo struct { + Remote string + Size int64 + IsDir bool +} + +var avInfos []avInfo + +func addVirtual(remote string, size int64, isDir bool) error { + avInfos = append(avInfos, avInfo{ + Remote: remote, + Size: size, + IsDir: isDir, + }) + return nil +} + func newTestCacheOpt(t *testing.T, opt vfscommon.Options) (r *fstest.Run, c *Cache, cleanup func()) { r = fstest.NewRun(t) ctx, cancel := context.WithCancel(context.Background()) - c, err := New(ctx, r.Fremote, &opt) + avInfos = nil + c, err := New(ctx, r.Fremote, &opt, addVirtual) require.NoError(t, err) cleanup = func() { diff --git a/vfs/vfscache/item.go b/vfs/vfscache/item.go index ca759ba67..9555a27ba 100644 --- a/vfs/vfscache/item.go +++ b/vfs/vfscache/item.go @@ -31,6 +31,7 @@ import ( // - Cache.toOSPathMeta // - Cache.mkdir // - Cache.objectFingerprint +// - Cache.AddVirtual // NB Item and downloader are tightly linked so it is necessary to // have a total lock ordering between them. downloader.mu must always @@ -654,6 +655,15 @@ func (item *Item) reload(ctx context.Context) error { if err != nil { return err } + // put the file into the directory listings + size, err := item._getSize() + if err != nil { + return errors.Wrap(err, "reload: failed to read size") + } + err = item.c.AddVirtual(item.name, size, false) + if err != nil { + return errors.Wrap(err, "reload: failed to add virtual dir entry") + } return nil } diff --git a/vfs/vfscache/item_test.go b/vfs/vfscache/item_test.go index 91fdf2137..933911ace 100644 --- a/vfs/vfscache/item_test.go +++ b/vfs/vfscache/item_test.go @@ -343,6 +343,11 @@ func TestItemReload(t *testing.T) { // And check the contents got written back to the remote checkObject(t, r, "existing", contents[:95]+"THEENDMYFRIEND") + + // And check that AddVirtual was called + assert.Equal(t, []avInfo{ + {Remote: "existing", Size: 109, IsDir: false}, + }, avInfos) } func TestItemReloadRemoteGone(t *testing.T) {