From ca19fd2d7e2480a3ee97e615bbb6b9ea2b261a23 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Fri, 27 Oct 2017 21:41:34 +0100 Subject: [PATCH] mountlib: Make read/write file handles support more standard interfaces Including Read, ReadAt, Seek, Close for read handles and Write, WriteAt, Close for read handles. --- cmd/cmount/fs.go | 8 ++- cmd/mount/read.go | 10 ++-- cmd/mount/write.go | 2 +- cmd/mountlib/read.go | 114 +++++++++++++++++++++++++++++++++++------- cmd/mountlib/write.go | 45 ++++++++++++++--- 5 files changed, 141 insertions(+), 38 deletions(-) diff --git a/cmd/cmount/fs.go b/cmd/cmount/fs.go index f9b454db4..8a0fb3a61 100644 --- a/cmd/cmount/fs.go +++ b/cmd/cmount/fs.go @@ -411,11 +411,10 @@ func (fsys *FS) Read(path string, buff []byte, ofst int64, fh uint64) (n int) { // Can only read from read file handle return -fuse.EIO } - data, err := rfh.Read(int64(len(buff)), ofst) + n, err := rfh.ReadAt(buff, ofst) if err != nil { return translateError(err) } - n = copy(buff, data) return n } @@ -431,12 +430,11 @@ func (fsys *FS) Write(path string, buff []byte, ofst int64, fh uint64) (n int) { // Can only write to write file handle return -fuse.EIO } - // FIXME made Write return int and Read take int since must fit in RAM - n64, err := wfh.Write(buff, ofst) + n, err := wfh.WriteAt(buff, ofst) if err != nil { return translateError(err) } - return int(n64) + return n } // Flush flushes an open file descriptor or path diff --git a/cmd/mount/read.go b/cmd/mount/read.go index b7a2cdec1..6548f87ab 100644 --- a/cmd/mount/read.go +++ b/cmd/mount/read.go @@ -29,14 +29,14 @@ var _ fusefs.HandleReader = (*ReadFileHandle)(nil) // Read from the file handle func (fh *ReadFileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) (err error) { - dataRead := -1 - defer fs.Trace(fh, "len=%d, offset=%d", req.Size, req.Offset)("read=%d, err=%v", &dataRead, &err) - data, err := fh.ReadFileHandle.Read(int64(req.Size), req.Offset) + var n int + defer fs.Trace(fh, "len=%d, offset=%d", req.Size, req.Offset)("read=%d, err=%v", &n, &err) + data := make([]byte, req.Size) + n, err = fh.ReadFileHandle.ReadAt(data, req.Offset) if err != nil { return translateError(err) } - resp.Data = data - dataRead = len(data) + resp.Data = data[:n] return nil } diff --git a/cmd/mount/write.go b/cmd/mount/write.go index efa16fef4..580e31a9d 100644 --- a/cmd/mount/write.go +++ b/cmd/mount/write.go @@ -28,7 +28,7 @@ var _ fusefs.HandleWriter = (*WriteFileHandle)(nil) // Write data to the file handle func (fh *WriteFileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse.WriteResponse) (err error) { defer fs.Trace(fh, "len=%d, offset=%d", len(req.Data), req.Offset)("written=%d, err=%v", &resp.Size, &err) - n, err := fh.WriteFileHandle.Write(req.Data, req.Offset) + n, err := fh.WriteFileHandle.WriteAt(req.Data, req.Offset) if err != nil { return translateError(err) } diff --git a/cmd/mountlib/read.go b/cmd/mountlib/read.go index fd7cf5408..752496234 100644 --- a/cmd/mountlib/read.go +++ b/cmd/mountlib/read.go @@ -14,14 +14,23 @@ type ReadFileHandle struct { closed bool // set if handle has been closed r *fs.Account o fs.Object - readCalled bool // set if read has been called - offset int64 + readCalled bool // set if read has been called + offset int64 // offset of read of o + roffset int64 // offset of Read() calls noSeek bool file *File hash *fs.MultiHasher opened bool } +// Check interfaces +var ( + _ io.Reader = (*ReadFileHandle)(nil) + _ io.ReaderAt = (*ReadFileHandle)(nil) + _ io.Seeker = (*ReadFileHandle)(nil) + _ io.Closer = (*ReadFileHandle)(nil) +) + func newReadFileHandle(f *File, o fs.Object) (*ReadFileHandle, error) { var hash *fs.MultiHasher var err error @@ -113,36 +122,72 @@ func (fh *ReadFileHandle) seek(offset int64, reopen bool) (err error) { return nil } -// Read from the file handle -func (fh *ReadFileHandle) Read(reqSize, reqOffset int64) (respData []byte, err error) { +// Seek the file +func (fh *ReadFileHandle) Seek(offset int64, whence int) (n int64, err error) { + size := fh.o.Size() + switch whence { + case 0: + fh.roffset = 0 + case 2: + fh.roffset = size + } + fh.roffset += offset + // we don't check the offset - the next Read will + return fh.roffset, nil +} + +// ReadAt reads len(p) bytes into p starting at offset off in the +// underlying input source. It returns the number of bytes read (0 <= +// n <= len(p)) and any error encountered. +// +// When ReadAt returns n < len(p), it returns a non-nil error +// explaining why more bytes were not returned. In this respect, +// ReadAt is stricter than Read. +// +// Even if ReadAt returns n < len(p), it may use all of p as scratch +// space during the call. If some data is available but not len(p) +// bytes, ReadAt blocks until either all the data is available or an +// error occurs. In this respect ReadAt is different from Read. +// +// If the n = len(p) bytes returned by ReadAt are at the end of the +// input source, ReadAt may return either err == EOF or err == nil. +// +// If ReadAt is reading from an input source with a seek offset, +// ReadAt should not affect nor be affected by the underlying seek +// offset. +// +// Clients of ReadAt can execute parallel ReadAt calls on the same +// input source. +// +// Implementations must not retain p. +func (fh *ReadFileHandle) ReadAt(p []byte, off int64) (n int, err error) { fh.mu.Lock() defer fh.mu.Unlock() err = fh.openPending() // FIXME pending open could be more efficient in the presense of seek (and retried) if err != nil { - return nil, err + return 0, err } - // fs.Debugf(fh.o, "ReadFileHandle.Read size %d offset %d", reqSize, reqOffset) + // fs.Debugf(fh.o, "ReadFileHandle.Read size %d offset %d", reqSize, off) if fh.closed { fs.Errorf(fh.o, "ReadFileHandle.Read error: %v", EBADF) - return nil, EBADF + return 0, EBADF } - doSeek := reqOffset != fh.offset - var n int + doSeek := off != fh.offset var newOffset int64 retries := 0 - buf := make([]byte, reqSize) + reqSize := len(p) doReopen := false for { if doSeek { // Are we attempting to seek beyond the end of the // file - if so just return EOF leaving the underlying // file in an unchanged state. - if reqOffset >= fh.o.Size() { - fs.Debugf(fh.o, "ReadFileHandle.Read attempt to read beyond end of file: %d > %d", reqOffset, fh.o.Size()) - return nil, nil + if off >= fh.o.Size() { + fs.Debugf(fh.o, "ReadFileHandle.Read attempt to read beyond end of file: %d > %d", off, fh.o.Size()) + return 0, nil } // Otherwise do the seek - err = fh.seek(reqOffset, doReopen) + err = fh.seek(off, doReopen) } else { err = nil } @@ -153,7 +198,7 @@ func (fh *ReadFileHandle) Read(reqSize, reqOffset int64) (respData []byte, err e // One exception to the above is if we fail to fully populate a // page cache page; a read into page cache is always page aligned. // Make sure we never serve a partial read, to avoid that. - n, err = io.ReadFull(fh.r, buf) + n, err = io.ReadFull(fh.r, p) newOffset = fh.offset + int64(n) // if err == nil && rand.Intn(10) == 0 { // err = errors.New("random error") @@ -177,19 +222,18 @@ func (fh *ReadFileHandle) Read(reqSize, reqOffset int64) (respData []byte, err e if err != nil { fs.Errorf(fh.o, "ReadFileHandle.Read error: %v", err) } else { - respData = buf[:n] fh.offset = newOffset // fs.Debugf(fh.o, "ReadFileHandle.Read OK") if fh.hash != nil { - _, err = fh.hash.Write(respData) + _, err = fh.hash.Write(p[:n]) if err != nil { fs.Errorf(fh.o, "ReadFileHandle.Read HashError: %v", err) - return nil, err + return 0, err } } } - return respData, err + return n, err } func (fh *ReadFileHandle) checkHash() error { @@ -210,6 +254,38 @@ func (fh *ReadFileHandle) checkHash() error { return nil } +// Read reads up to len(p) bytes into p. It returns the number of bytes read (0 +// <= n <= len(p)) and any error encountered. Even if Read returns n < len(p), +// it may use all of p as scratch space during the call. If some data is +// available but not len(p) bytes, Read conventionally returns what is +// available instead of waiting for more. +// +// When Read encounters an error or end-of-file condition after successfully +// reading n > 0 bytes, it returns the number of bytes read. It may return the +// (non-nil) error from the same call or return the error (and n == 0) from a +// subsequent call. An instance of this general case is that a Reader returning +// a non-zero number of bytes at the end of the input stream may return either +// err == EOF or err == nil. The next Read should return 0, EOF. +// +// Callers should always process the n > 0 bytes returned before considering +// the error err. Doing so correctly handles I/O errors that happen after +// reading some bytes and also both of the allowed EOF behaviors. +// +// Implementations of Read are discouraged from returning a zero byte count +// with a nil error, except when len(p) == 0. Callers should treat a return of +// 0 and nil as indicating that nothing happened; in particular it does not +// indicate EOF. +// +// Implementations must not retain p. +func (fh *ReadFileHandle) Read(p []byte) (n int, err error) { + if fh.roffset >= fh.o.Size() { + return 0, io.EOF + } + n, err = fh.ReadAt(p, fh.roffset) + fh.roffset += int64(n) + return n, err +} + // close the file handle returning EBADF if it has been // closed already. // diff --git a/cmd/mountlib/write.go b/cmd/mountlib/write.go index f17b3c6ca..c3f4c8c51 100644 --- a/cmd/mountlib/write.go +++ b/cmd/mountlib/write.go @@ -21,6 +21,13 @@ type WriteFileHandle struct { offset int64 } +// Check interfaces +var ( + _ io.Writer = (*WriteFileHandle)(nil) + _ io.WriterAt = (*WriteFileHandle)(nil) + _ io.Closer = (*WriteFileHandle)(nil) +) + func newWriteFileHandle(d *Dir, f *File, src fs.ObjectInfo) (*WriteFileHandle, error) { fh := &WriteFileHandle{ remote: src.Remote(), @@ -60,12 +67,23 @@ func (fh *WriteFileHandle) Node() Node { return fh.file } -// Write data to the file handle -func (fh *WriteFileHandle) Write(data []byte, offset int64) (written int64, err error) { - // fs.Debugf(fh.remote, "WriteFileHandle.Write len=%d", len(data)) +// WriteAt writes len(p) bytes from p to the underlying data stream at offset +// off. It returns the number of bytes written from p (0 <= n <= len(p)) and +// any error encountered that caused the write to stop early. WriteAt must +// return a non-nil error if it returns n < len(p). +// +// If WriteAt is writing to a destination with a seek offset, WriteAt should +// not affect nor be affected by the underlying seek offset. +// +// Clients of WriteAt can execute parallel WriteAt calls on the same +// destination if the ranges do not overlap. +// +// Implementations must not retain p. +func (fh *WriteFileHandle) WriteAt(p []byte, off int64) (n int, err error) { + // fs.Debugf(fh.remote, "WriteFileHandle.Write len=%d", len(p)) fh.mu.Lock() defer fh.mu.Unlock() - if fh.offset != offset { + if fh.offset != off { fs.Errorf(fh.remote, "WriteFileHandle.Write can't seek in file") return 0, ESPIPE } @@ -74,16 +92,27 @@ func (fh *WriteFileHandle) Write(data []byte, offset int64) (written int64, err return 0, EBADF } fh.writeCalled = true - n, err := fh.pipeWriter.Write(data) - written = int64(n) - fh.offset += written + n, err = fh.pipeWriter.Write(p) + fh.offset += int64(n) fh.file.setSize(fh.offset) if err != nil { fs.Errorf(fh.remote, "WriteFileHandle.Write error: %v", err) return 0, err } // fs.Debugf(fh.remote, "WriteFileHandle.Write OK (%d bytes written)", n) - return written, nil + return n, nil +} + +// Write writes len(p) bytes from p to the underlying data stream. It returns +// the number of bytes written from p (0 <= n <= len(p)) and any error +// encountered that caused the write to stop early. Write must return a non-nil +// error if it returns n < len(p). Write must not modify the slice data, even +// temporarily. +// +// Implementations must not retain p. +func (fh *WriteFileHandle) Write(p []byte) (n int, err error) { + // Since we can't seek, just call WriteAt with the current offset + return fh.WriteAt(p, fh.offset) } // Offset returns the offset of the file pointer