diff --git a/vfs/help.go b/vfs/help.go index 4e8be3bb3..4b1cb3fea 100644 --- a/vfs/help.go +++ b/vfs/help.go @@ -159,6 +159,13 @@ the data that has been downloaded present in them. This mode should support all normal file system operations and is otherwise identical to --vfs-cache-mode writes. +When reading a file rclone will read --buffer-size plus +--vfs-read-ahead bytes ahead. The --buffer-size is buffered in memory +whereas the --vfs-read-ahead is buffered on disk. + +When using this mode it is recommended that --buffer-size is not set +too big and --vfs-read-ahead is set large if required. + ### VFS Performance These flags may be used to enable/disable features of the VFS for diff --git a/vfs/vfscache/downloaders/downloaders.go b/vfs/vfscache/downloaders/downloaders.go index d8dfba5af..08f230bab 100644 --- a/vfs/vfscache/downloaders/downloaders.go +++ b/vfs/vfscache/downloaders/downloaders.go @@ -273,9 +273,14 @@ func (dls *Downloaders) _closeWaiters(err error) { // // call with lock held func (dls *Downloaders) _ensureDownloader(r ranges.Range) (err error) { - // FIXME this window could be a different config var? + // The window includes potentially unread data in the buffer window := int64(fs.Config.BufferSize) + // Increase the read range by the read ahead if set + if dls.opt.ReadAhead > 0 { + r.Size += int64(dls.opt.ReadAhead) + } + // We may be reopening a downloader after a failure here or // doing a tentative prefetch so check to see that we haven't // read some stuff already. @@ -310,15 +315,15 @@ func (dls *Downloaders) _ensureDownloader(r ranges.Range) (err error) { // If there isn't one then start a new one dls._removeClosed() for _, dl = range dls.dls { - start, maxOffset := dl.getRange() + start, offset := dl.getRange() // The downloader's offset to offset+window is the gap // in which we would like to re-use this // downloader. The downloader will never reach before - // start and maxOffset+windows is too far away - we'd + // start and offset+windows is too far away - we'd // rather start another downloader. - // fs.Debugf(nil, "r=%v start=%d, maxOffset=%d, found=%v", r, start, maxOffset, r.Pos >= start && r.Pos < maxOffset+window) - if r.Pos >= start && r.Pos < maxOffset+window { + // fs.Debugf(nil, "r=%v start=%d, offset=%d, found=%v", r, start, offset, r.Pos >= start && r.Pos < offset+window) + if r.Pos >= start && r.Pos < offset+window { // Found downloader which will soon have our data dl.setRange(r) return nil @@ -598,8 +603,8 @@ func (dl *downloader) setRange(r ranges.Range) { } // get the current range this downloader is working on -func (dl *downloader) getRange() (start, maxOffset int64) { +func (dl *downloader) getRange() (start, offset int64) { dl.mu.Lock() defer dl.mu.Unlock() - return dl.start, dl.maxOffset + return dl.start, dl.offset } diff --git a/vfs/vfscommon/options.go b/vfs/vfscommon/options.go index 047888753..948c66d21 100644 --- a/vfs/vfscommon/options.go +++ b/vfs/vfscommon/options.go @@ -31,6 +31,7 @@ type Options struct { WriteWait time.Duration // time to wait for in-sequence write ReadWait time.Duration // time to wait for in-sequence read WriteBack time.Duration // time to wait before writing back dirty files + ReadAhead fs.SizeSuffix // bytes to read ahead in cache mode "full" } // DefaultOpt is the default values uses for Opt @@ -56,4 +57,5 @@ var DefaultOpt = Options{ WriteWait: 1000 * time.Millisecond, ReadWait: 20 * time.Millisecond, WriteBack: 5 * time.Second, + ReadAhead: 0 * fs.MebiByte, } diff --git a/vfs/vfsflags/vfsflags.go b/vfs/vfsflags/vfsflags.go index 2185673ab..ebc1ead16 100644 --- a/vfs/vfsflags/vfsflags.go +++ b/vfs/vfsflags/vfsflags.go @@ -36,5 +36,6 @@ func AddFlags(flagSet *pflag.FlagSet) { flags.DurationVarP(flagSet, &Opt.WriteWait, "vfs-write-wait", "", Opt.WriteWait, "Time to wait for in-sequence write before giving error.") flags.DurationVarP(flagSet, &Opt.ReadWait, "vfs-read-wait", "", Opt.ReadWait, "Time to wait for in-sequence read before seeking.") flags.DurationVarP(flagSet, &Opt.WriteBack, "vfs-write-back", "", Opt.WriteBack, "Time to writeback files after last use when using cache.") + flags.FVarP(flagSet, &Opt.ReadAhead, "vfs-read-ahead", "", "Extra read ahead over --buffer-size when using cache-mode full.") platformFlags(flagSet) }