vfs: Add recovered items on cache reload to directory listings

Before this change, if we restarted an upload after a restart then the
file would get uploaded but never added to the directory listings.

This change makes sure we add virtual items to the directory cache
when reloading the cache so that they show up properly.

vfs/vfs.go

@@ -259,7 +259,7 @@ func (vfs *VFS) SetCacheMode(cacheMode vfscommon.CacheMode) {
 	vfs.cache = nil
 	if cacheMode > vfscommon.CacheModeOff {
 		ctx, cancel := context.WithCancel(context.Background())
-		cache, err := vfscache.New(ctx, vfs.f, &vfs.Opt) // FIXME pass on context or get from Opt?
+		cache, err := vfscache.New(ctx, vfs.f, &vfs.Opt, vfs.AddVirtual) // FIXME pass on context or get from Opt?
 		if err != nil {
 			fs.Errorf(nil, "Failed to create vfs cache - disabling: %v", err)
 			vfs.Opt.CacheMode = vfscommon.CacheModeOff
@@ -652,3 +652,13 @@ func (vfs *VFS) ReadFile(filename string) (b []byte, err error) {
 	defer fs.CheckClose(f, &err)
 	return ioutil.ReadAll(f)
 }
+
+// AddVirtual adds the object (file or dir) to the directory cache
+func (vfs *VFS) AddVirtual(remote string, size int64, isDir bool) error {
+	dir, leaf, err := vfs.StatParent(remote)
+	if err != nil {
+		return err
+	}
+	dir.AddVirtual(leaf, size, false)
+	return nil
+}

vfs/vfscache/cache.go

@@ -45,17 +45,26 @@ type Cache struct {
 	hashType   hash.Type            // hash to use locally and remotely
 	hashOption *fs.HashesOption     // corresponding OpenOption
 	writeback  *writeback.WriteBack // holds Items for writeback
+	avFn       AddVirtualFn         // if set, can be called to add dir entries
 
 	mu   sync.Mutex       // protects the following variables
 	item map[string]*Item // files/directories in the cache
 	used int64            // total size of files in the cache
 }
 
+// AddVirtualFn if registered by the WithAddVirtual method, can be
+// called to register the object or directory at remote as a virtual
+// entry in directory listings.
+//
+// This is used when reloading the Cache and uploading items need to
+// go into the directory tree.
+type AddVirtualFn func(remote string, size int64, isDir bool) error
+
 // New creates a new cache heirachy for fremote
 //
 // This starts background goroutines which can be cancelled with the
 // context passed in.
-func New(ctx context.Context, fremote fs.Fs, opt *vfscommon.Options) (*Cache, error) {
+func New(ctx context.Context, fremote fs.Fs, opt *vfscommon.Options, avFn AddVirtualFn) (*Cache, error) {
 	fRoot := filepath.FromSlash(fremote.Root())
 	if runtime.GOOS == "windows" {
 		if strings.HasPrefix(fRoot, `\\?`) {
@@ -90,6 +99,7 @@ func New(ctx context.Context, fremote fs.Fs, opt *vfscommon.Options) (*Cache, er
 		hashType:   hashType,
 		hashOption: hashOption,
 		writeback:  writeback.New(ctx, opt),
+		avFn:       avFn,
 	}
 
 	// Make sure cache directories exist
@@ -567,3 +577,12 @@ func (c *Cache) Dump() string {
 	out.WriteString("}\n")
 	return out.String()
 }
+
+// AddVirtual adds a virtual directory entry by calling the addVirtual
+// callback if one has been registered.
+func (c *Cache) AddVirtual(remote string, size int64, isDir bool) error {
+	if c.avFn == nil {
+		return errors.New("no AddVirtual function registered")
+	}
+	return c.avFn(remote, size, isDir)
+}

vfs/vfscache/cache_test.go

@@ -51,12 +51,30 @@ func assertPathExist(t *testing.T, path string) os.FileInfo {
 	return fi
 }
 
+type avInfo struct {
+	Remote string
+	Size   int64
+	IsDir  bool
+}
+
+var avInfos []avInfo
+
+func addVirtual(remote string, size int64, isDir bool) error {
+	avInfos = append(avInfos, avInfo{
+		Remote: remote,
+		Size:   size,
+		IsDir:  isDir,
+	})
+	return nil
+}
+
 func newTestCacheOpt(t *testing.T, opt vfscommon.Options) (r *fstest.Run, c *Cache, cleanup func()) {
 	r = fstest.NewRun(t)
 
 	ctx, cancel := context.WithCancel(context.Background())
 
-	c, err := New(ctx, r.Fremote, &opt)
+	avInfos = nil
+	c, err := New(ctx, r.Fremote, &opt, addVirtual)
 	require.NoError(t, err)
 
 	cleanup = func() {

vfs/vfscache/item.go

@@ -31,6 +31,7 @@ import (
 // - Cache.toOSPathMeta
 // - Cache.mkdir
 // - Cache.objectFingerprint
+// - Cache.AddVirtual
 
 // NB Item and downloader are tightly linked so it is necessary to
 // have a total lock ordering between them. downloader.mu must always
@@ -654,6 +655,15 @@ func (item *Item) reload(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
+	// put the file into the directory listings
+	size, err := item._getSize()
+	if err != nil {
+		return errors.Wrap(err, "reload: failed to read size")
+	}
+	err = item.c.AddVirtual(item.name, size, false)
+	if err != nil {
+		return errors.Wrap(err, "reload: failed to add virtual dir entry")
+	}
 	return nil
 }
 

vfs/vfscache/item_test.go

@@ -343,6 +343,11 @@ func TestItemReload(t *testing.T) {
 
 	// And check the contents got written back to the remote
 	checkObject(t, r, "existing", contents[:95]+"THEENDMYFRIEND")
+
+	// And check that AddVirtual was called
+	assert.Equal(t, []avInfo{
+		{Remote: "existing", Size: 109, IsDir: false},
+	}, avInfos)
 }
 
 func TestItemReloadRemoteGone(t *testing.T) {