vfs: make objects of unknown size readable through the VFS

These objects (eg Google Docs) appear with 0 length in the VFS.

Before this change, these only read 0 bytes.

After this change, even though the size appears to be 0, the objects
can be read to the end.  If the objects are read to the end then the
size on the handle will be updated.
This commit is contained in:
Nick Craig-Wood 2019-09-14 13:09:07 +01:00
parent 2e80e035c9
commit ba121eddf0
2 changed files with 75 additions and 20 deletions

View File

@ -7,6 +7,8 @@ import (
"testing"
"github.com/rclone/rclone/fstest"
"github.com/rclone/rclone/fstest/mockfs"
"github.com/rclone/rclone/fstest/mockobject"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@ -110,6 +112,51 @@ func TestFileOpenRead(t *testing.T) {
require.NoError(t, fd.Close())
}
func TestFileOpenReadUnknownSize(t *testing.T) {
var (
contents = []byte("file contents")
remote = "file.txt"
ctx = context.Background()
)
// create a mock object which returns size -1
o := mockobject.New(remote).WithContent(contents, mockobject.SeekModeNone)
o.SetUnknownSize(true)
assert.Equal(t, int64(-1), o.Size())
// add it to a mock fs
f := mockfs.NewFs("test", "root")
f.AddObject(o)
testObj, err := f.NewObject(ctx, remote)
require.NoError(t, err)
assert.Equal(t, int64(-1), testObj.Size())
// create a VFS from that mockfs
vfs := New(f, nil)
// find the file
node, err := vfs.Stat(remote)
require.NoError(t, err)
require.True(t, node.IsFile())
file := node.(*File)
// open it
fd, err := file.openRead()
require.NoError(t, err)
assert.Equal(t, int64(0), fd.Size())
// check the contents are not empty even though size is empty
gotContents, err := ioutil.ReadAll(fd)
require.NoError(t, err)
assert.Equal(t, contents, gotContents)
t.Logf("gotContents = %q", gotContents)
// check that file size has been updated
assert.Equal(t, int64(len(contents)), fd.Size())
require.NoError(t, fd.Close())
}
func TestFileOpenWrite(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()

View File

@ -16,19 +16,20 @@ import (
// ReadFileHandle is an open for read file handle on a File
type ReadFileHandle struct {
baseHandle
done func(err error)
mu sync.Mutex
closed bool // set if handle has been closed
r *accounting.Account
readCalled bool // set if read has been called
size int64 // size of the object
offset int64 // offset of read of o
roffset int64 // offset of Read() calls
noSeek bool
file *File
hash *hash.MultiHasher
opened bool
remote string
done func(err error)
mu sync.Mutex
closed bool // set if handle has been closed
r *accounting.Account
readCalled bool // set if read has been called
size int64 // size of the object (0 for unknown length)
offset int64 // offset of read of o
roffset int64 // offset of Read() calls
noSeek bool
sizeUnknown bool // set if size of source is not known
file *File
hash *hash.MultiHasher
opened bool
remote string
}
// Check interfaces
@ -51,11 +52,12 @@ func newReadFileHandle(f *File) (*ReadFileHandle, error) {
}
fh := &ReadFileHandle{
remote: o.Remote(),
noSeek: f.d.vfs.Opt.NoSeek,
file: f,
hash: mhash,
size: nonNegative(o.Size()),
remote: o.Remote(),
noSeek: f.d.vfs.Opt.NoSeek,
file: f,
hash: mhash,
size: nonNegative(o.Size()),
sizeUnknown: o.Size() < 0,
}
return fh, nil
}
@ -208,6 +210,7 @@ func (fh *ReadFileHandle) ReadAt(p []byte, off int64) (n int, err error) {
// Implementation of ReadAt - call with lock held
func (fh *ReadFileHandle) readAt(p []byte, off int64) (n int, err error) {
// defer log.Trace(fh.remote, "p[%d], off=%d", len(p), off)("n=%d, err=%v", &n, &err)
err = fh.openPending() // FIXME pending open could be more efficient in the presense of seek (and retries)
if err != nil {
return 0, err
@ -250,7 +253,12 @@ func (fh *ReadFileHandle) readAt(p []byte, off int64) (n int, err error) {
// }
if err == nil {
break
} else if (err == io.ErrUnexpectedEOF || err == io.EOF) && newOffset == fh.size {
} else if (err == io.ErrUnexpectedEOF || err == io.EOF) && (newOffset == fh.size || fh.sizeUnknown) {
if fh.sizeUnknown {
// size is now known since we have read to the end
fh.sizeUnknown = false
fh.size = newOffset
}
// Have read to end of file - reset error
err = nil
break
@ -331,7 +339,7 @@ func (fh *ReadFileHandle) checkHash() error {
func (fh *ReadFileHandle) Read(p []byte) (n int, err error) {
fh.mu.Lock()
defer fh.mu.Unlock()
if fh.roffset >= fh.size {
if fh.roffset >= fh.size && !fh.sizeUnknown {
return 0, io.EOF
}
n, err = fh.readAt(p, fh.roffset)