diff --git a/vfs/file_test.go b/vfs/file_test.go index c451f2d6c..06219b737 100644 --- a/vfs/file_test.go +++ b/vfs/file_test.go @@ -7,6 +7,8 @@ import ( "testing" "github.com/rclone/rclone/fstest" + "github.com/rclone/rclone/fstest/mockfs" + "github.com/rclone/rclone/fstest/mockobject" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -110,6 +112,51 @@ func TestFileOpenRead(t *testing.T) { require.NoError(t, fd.Close()) } +func TestFileOpenReadUnknownSize(t *testing.T) { + var ( + contents = []byte("file contents") + remote = "file.txt" + ctx = context.Background() + ) + + // create a mock object which returns size -1 + o := mockobject.New(remote).WithContent(contents, mockobject.SeekModeNone) + o.SetUnknownSize(true) + assert.Equal(t, int64(-1), o.Size()) + + // add it to a mock fs + f := mockfs.NewFs("test", "root") + f.AddObject(o) + testObj, err := f.NewObject(ctx, remote) + require.NoError(t, err) + assert.Equal(t, int64(-1), testObj.Size()) + + // create a VFS from that mockfs + vfs := New(f, nil) + + // find the file + node, err := vfs.Stat(remote) + require.NoError(t, err) + require.True(t, node.IsFile()) + file := node.(*File) + + // open it + fd, err := file.openRead() + require.NoError(t, err) + assert.Equal(t, int64(0), fd.Size()) + + // check the contents are not empty even though size is empty + gotContents, err := ioutil.ReadAll(fd) + require.NoError(t, err) + assert.Equal(t, contents, gotContents) + t.Logf("gotContents = %q", gotContents) + + // check that file size has been updated + assert.Equal(t, int64(len(contents)), fd.Size()) + + require.NoError(t, fd.Close()) +} + func TestFileOpenWrite(t *testing.T) { r := fstest.NewRun(t) defer r.Finalise() diff --git a/vfs/read.go b/vfs/read.go index af6b3b487..cd65f0e08 100644 --- a/vfs/read.go +++ b/vfs/read.go @@ -16,19 +16,20 @@ import ( // ReadFileHandle is an open for read file handle on a File type ReadFileHandle struct { baseHandle - done func(err error) - mu sync.Mutex - closed bool // set if handle has been closed - r *accounting.Account - readCalled bool // set if read has been called - size int64 // size of the object - offset int64 // offset of read of o - roffset int64 // offset of Read() calls - noSeek bool - file *File - hash *hash.MultiHasher - opened bool - remote string + done func(err error) + mu sync.Mutex + closed bool // set if handle has been closed + r *accounting.Account + readCalled bool // set if read has been called + size int64 // size of the object (0 for unknown length) + offset int64 // offset of read of o + roffset int64 // offset of Read() calls + noSeek bool + sizeUnknown bool // set if size of source is not known + file *File + hash *hash.MultiHasher + opened bool + remote string } // Check interfaces @@ -51,11 +52,12 @@ func newReadFileHandle(f *File) (*ReadFileHandle, error) { } fh := &ReadFileHandle{ - remote: o.Remote(), - noSeek: f.d.vfs.Opt.NoSeek, - file: f, - hash: mhash, - size: nonNegative(o.Size()), + remote: o.Remote(), + noSeek: f.d.vfs.Opt.NoSeek, + file: f, + hash: mhash, + size: nonNegative(o.Size()), + sizeUnknown: o.Size() < 0, } return fh, nil } @@ -208,6 +210,7 @@ func (fh *ReadFileHandle) ReadAt(p []byte, off int64) (n int, err error) { // Implementation of ReadAt - call with lock held func (fh *ReadFileHandle) readAt(p []byte, off int64) (n int, err error) { + // defer log.Trace(fh.remote, "p[%d], off=%d", len(p), off)("n=%d, err=%v", &n, &err) err = fh.openPending() // FIXME pending open could be more efficient in the presense of seek (and retries) if err != nil { return 0, err @@ -250,7 +253,12 @@ func (fh *ReadFileHandle) readAt(p []byte, off int64) (n int, err error) { // } if err == nil { break - } else if (err == io.ErrUnexpectedEOF || err == io.EOF) && newOffset == fh.size { + } else if (err == io.ErrUnexpectedEOF || err == io.EOF) && (newOffset == fh.size || fh.sizeUnknown) { + if fh.sizeUnknown { + // size is now known since we have read to the end + fh.sizeUnknown = false + fh.size = newOffset + } // Have read to end of file - reset error err = nil break @@ -331,7 +339,7 @@ func (fh *ReadFileHandle) checkHash() error { func (fh *ReadFileHandle) Read(p []byte) (n int, err error) { fh.mu.Lock() defer fh.mu.Unlock() - if fh.roffset >= fh.size { + if fh.roffset >= fh.size && !fh.sizeUnknown { return 0, io.EOF } n, err = fh.readAt(p, fh.roffset)