Add mount command to implement FUSE mounting of remotes #494
This enables any rclone remote to be mounted and used as a filesystem with some limitations. Only supported for Linux, FreeBSD and OS X
This commit is contained in:
Nick Craig-Wood
parent
d7b79b4481
commit
f22029bf3d
|
|
@ -19,6 +19,7 @@ import (
|
|||
_ "github.com/ncw/rclone/cmd/md5sum"
|
||||
_ "github.com/ncw/rclone/cmd/memtest"
|
||||
_ "github.com/ncw/rclone/cmd/mkdir"
|
||||
_ "github.com/ncw/rclone/cmd/mount"
|
||||
_ "github.com/ncw/rclone/cmd/move"
|
||||
_ "github.com/ncw/rclone/cmd/purge"
|
||||
_ "github.com/ncw/rclone/cmd/rmdir"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,57 @@
|
|||
// +build linux darwin freebsd
|
||||
|
||||
package mount
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/ncw/rclone/fs"
|
||||
)
|
||||
|
||||
// info to create a new object
|
||||
type createInfo struct {
|
||||
f fs.Fs
|
||||
remote string
|
||||
}
|
||||
|
||||
func newCreateInfo(f fs.Fs, remote string) *createInfo {
|
||||
return &createInfo{
|
||||
f: f,
|
||||
remote: remote,
|
||||
}
|
||||
}
|
||||
|
||||
// Fs returns read only access to the Fs that this object is part of
|
||||
func (ci *createInfo) Fs() fs.Info {
|
||||
return ci.f
|
||||
}
|
||||
|
||||
// Remote returns the remote path
|
||||
func (ci *createInfo) Remote() string {
|
||||
return ci.remote
|
||||
}
|
||||
|
||||
// Hash returns the selected checksum of the file
|
||||
// If no checksum is available it returns ""
|
||||
func (ci *createInfo) Hash(fs.HashType) (string, error) {
|
||||
return "", fs.ErrHashUnsupported
|
||||
}
|
||||
|
||||
// ModTime returns the modification date of the file
|
||||
// It should return a best guess if one isn't available
|
||||
func (ci *createInfo) ModTime() time.Time {
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
// Size returns the size of the file
|
||||
func (ci *createInfo) Size() int64 {
|
||||
// FIXME this means this won't work with all remotes...
|
||||
return 0
|
||||
}
|
||||
|
||||
// Storable says whether this object can be stored
|
||||
func (ci *createInfo) Storable() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
var _ fs.ObjectInfo = (*createInfo)(nil)
|
||||
|
|
@ -0,0 +1,377 @@
|
|||
// +build linux darwin freebsd
|
||||
|
||||
package mount
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"bazil.org/fuse"
|
||||
fusefs "bazil.org/fuse/fs"
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/pkg/errors"
|
||||
"golang.org/x/net/context"
|
||||
)
|
||||
|
||||
// DirEntry describes the contents of a directory entry
|
||||
//
|
||||
// It can be a file or a directory
|
||||
//
|
||||
// node may be nil, but o may not
|
||||
type DirEntry struct {
|
||||
o fs.BasicInfo
|
||||
node fusefs.Node
|
||||
}
|
||||
|
||||
// Dir represents a directory entry
|
||||
type Dir struct {
|
||||
f fs.Fs
|
||||
path string
|
||||
mu sync.RWMutex // protects the following
|
||||
read bool
|
||||
items map[string]*DirEntry
|
||||
}
|
||||
|
||||
func newDir(f fs.Fs, path string) *Dir {
|
||||
return &Dir{
|
||||
f: f,
|
||||
path: path,
|
||||
}
|
||||
}
|
||||
|
||||
// addObject adds a new object or directory to the directory
|
||||
//
|
||||
// note that we add new objects rather than updating old ones
|
||||
func (d *Dir) addObject(o fs.BasicInfo, node fusefs.Node) *DirEntry {
|
||||
item := &DirEntry{
|
||||
o: o,
|
||||
node: node,
|
||||
}
|
||||
d.mu.Lock()
|
||||
d.items[path.Base(o.Remote())] = item
|
||||
d.mu.Unlock()
|
||||
return item
|
||||
}
|
||||
|
||||
// delObject removes an object from the directory
|
||||
func (d *Dir) delObject(leaf string) {
|
||||
d.mu.Lock()
|
||||
delete(d.items, leaf)
|
||||
d.mu.Unlock()
|
||||
}
|
||||
|
||||
// read the directory
|
||||
func (d *Dir) readDir() error {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
if d.read {
|
||||
return nil
|
||||
}
|
||||
objs, dirs, err := fs.NewLister().SetLevel(1).Start(d.f, d.path).GetAll()
|
||||
if err == fs.ErrorDirNotFound {
|
||||
// We treat directory not found as empty because we
|
||||
// create directories on the fly
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
// Cache the items by name
|
||||
d.items = make(map[string]*DirEntry, len(objs)+len(dirs))
|
||||
for _, obj := range objs {
|
||||
name := path.Base(obj.Remote())
|
||||
d.items[name] = &DirEntry{
|
||||
o: obj,
|
||||
node: nil,
|
||||
}
|
||||
}
|
||||
for _, dir := range dirs {
|
||||
name := path.Base(dir.Remote())
|
||||
d.items[name] = &DirEntry{
|
||||
o: dir,
|
||||
node: nil,
|
||||
}
|
||||
}
|
||||
d.read = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// lookup a single item in the directory
|
||||
//
|
||||
// returns fuse.ENOENT if not found.
|
||||
func (d *Dir) lookup(leaf string) (*DirEntry, error) {
|
||||
err := d.readDir()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
d.mu.RLock()
|
||||
item, ok := d.items[leaf]
|
||||
d.mu.RUnlock()
|
||||
if !ok {
|
||||
return nil, fuse.ENOENT
|
||||
}
|
||||
return item, nil
|
||||
}
|
||||
|
||||
// Check to see if a directory is empty
|
||||
func (d *Dir) isEmpty() (bool, error) {
|
||||
err := d.readDir()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
d.mu.RLock()
|
||||
defer d.mu.RUnlock()
|
||||
return len(d.items) == 0, nil
|
||||
}
|
||||
|
||||
// Check interface satsified
|
||||
var _ fusefs.Node = (*Dir)(nil)
|
||||
|
||||
// Attr updates the attribes of a directory
|
||||
func (d *Dir) Attr(ctx context.Context, a *fuse.Attr) error {
|
||||
fs.Debug(d.path, "Dir.Attr")
|
||||
a.Mode = os.ModeDir | dirPerms
|
||||
// FIXME include Valid so get some caching? Also mtime
|
||||
return nil
|
||||
}
|
||||
|
||||
// lookupNode calls lookup then makes sure the node is not nil in the DirEntry
|
||||
func (d *Dir) lookupNode(leaf string) (item *DirEntry, err error) {
|
||||
item, err = d.lookup(leaf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if item.node != nil {
|
||||
return item, nil
|
||||
}
|
||||
var node fusefs.Node
|
||||
switch x := item.o.(type) {
|
||||
case fs.Object:
|
||||
node, err = newFile(d, x), nil
|
||||
case *fs.Dir:
|
||||
node, err = newDir(d.f, x.Remote()), nil
|
||||
default:
|
||||
err = errors.Errorf("unknown type %T", item)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
item = d.addObject(item.o, node)
|
||||
return item, err
|
||||
}
|
||||
|
||||
// Check interface satisfied
|
||||
var _ fusefs.NodeRequestLookuper = (*Dir)(nil)
|
||||
|
||||
// Lookup looks up a specific entry in the receiver.
|
||||
//
|
||||
// Lookup should return a Node corresponding to the entry. If the
|
||||
// name does not exist in the directory, Lookup should return ENOENT.
|
||||
//
|
||||
// Lookup need not to handle the names "." and "..".
|
||||
func (d *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.LookupResponse) (node fusefs.Node, err error) {
|
||||
path := path.Join(d.path, req.Name)
|
||||
fs.Debug(path, "Dir.Lookup")
|
||||
item, err := d.lookupNode(req.Name)
|
||||
if err != nil {
|
||||
if err != fuse.ENOENT {
|
||||
fs.ErrorLog(path, "Dir.Lookup error: %v", err)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
fs.Debug(path, "Dir.Lookup OK")
|
||||
return item.node, nil
|
||||
}
|
||||
|
||||
// Check interface satisfied
|
||||
var _ fusefs.HandleReadDirAller = (*Dir)(nil)
|
||||
|
||||
// ReadDirAll reads the contents of the directory
|
||||
func (d *Dir) ReadDirAll(ctx context.Context) (dirents []fuse.Dirent, err error) {
|
||||
fs.Debug(d.path, "Dir.ReadDirAll")
|
||||
err = d.readDir()
|
||||
if err != nil {
|
||||
fs.Debug(d.path, "Dir.ReadDirAll error: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
d.mu.RLock()
|
||||
defer d.mu.RUnlock()
|
||||
for _, item := range d.items {
|
||||
var dirent fuse.Dirent
|
||||
switch x := item.o.(type) {
|
||||
case fs.Object:
|
||||
dirent = fuse.Dirent{
|
||||
// Inode FIXME ???
|
||||
Type: fuse.DT_File,
|
||||
Name: path.Base(x.Remote()),
|
||||
}
|
||||
case *fs.Dir:
|
||||
dirent = fuse.Dirent{
|
||||
// Inode FIXME ???
|
||||
Type: fuse.DT_Dir,
|
||||
Name: path.Base(x.Remote()),
|
||||
}
|
||||
default:
|
||||
err = errors.Errorf("unknown type %T", item)
|
||||
fs.ErrorLog(d.path, "Dir.ReadDirAll error: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
dirents = append(dirents, dirent)
|
||||
}
|
||||
fs.Debug(d.path, "Dir.ReadDirAll OK with %d entries", len(dirents))
|
||||
return dirents, nil
|
||||
}
|
||||
|
||||
var _ fusefs.NodeCreater = (*Dir)(nil)
|
||||
|
||||
// Create makes a new file
|
||||
func (d *Dir) Create(ctx context.Context, req *fuse.CreateRequest, resp *fuse.CreateResponse) (fusefs.Node, fusefs.Handle, error) {
|
||||
path := path.Join(d.path, req.Name)
|
||||
fs.Debug(path, "Dir.Create")
|
||||
src := newCreateInfo(d.f, path)
|
||||
// This gets added to the directory when the file is written
|
||||
file := newFile(d, nil)
|
||||
fh, err := newWriteFileHandle(d, file, src)
|
||||
if err != nil {
|
||||
fs.ErrorLog(path, "Dir.Create error: %v", err)
|
||||
return nil, nil, err
|
||||
}
|
||||
fs.Debug(path, "Dir.Create OK")
|
||||
return file, fh, nil
|
||||
}
|
||||
|
||||
var _ fusefs.NodeMkdirer = (*Dir)(nil)
|
||||
|
||||
// Mkdir creates a new directory
|
||||
func (d *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fusefs.Node, error) {
|
||||
// We just pretend to have created the directory - rclone will
|
||||
// actually create the directory if we write files into it
|
||||
path := path.Join(d.path, req.Name)
|
||||
fs.Debug(path, "Dir.Mkdir")
|
||||
fsDir := &fs.Dir{
|
||||
Name: path,
|
||||
When: time.Now(),
|
||||
}
|
||||
dir := newDir(d.f, path)
|
||||
d.addObject(fsDir, dir)
|
||||
fs.Debug(path, "Dir.Mkdir OK")
|
||||
return dir, nil
|
||||
}
|
||||
|
||||
var _ fusefs.NodeRemover = (*Dir)(nil)
|
||||
|
||||
// Remove removes the entry with the given name from
|
||||
// the receiver, which must be a directory. The entry to be removed
|
||||
// may correspond to a file (unlink) or to a directory (rmdir).
|
||||
func (d *Dir) Remove(ctx context.Context, req *fuse.RemoveRequest) error {
|
||||
path := path.Join(d.path, req.Name)
|
||||
fs.Debug(path, "Dir.Remove")
|
||||
item, err := d.lookupNode(req.Name)
|
||||
if err != nil {
|
||||
fs.ErrorLog(path, "Dir.Remove error: %v", err)
|
||||
return err
|
||||
}
|
||||
switch x := item.o.(type) {
|
||||
case fs.Object:
|
||||
err = x.Remove()
|
||||
if err != nil {
|
||||
fs.ErrorLog(path, "Dir.Remove file error: %v", err)
|
||||
return err
|
||||
}
|
||||
case *fs.Dir:
|
||||
// Do nothing for deleting directory - rclone can't
|
||||
// currently remote a random directory
|
||||
//
|
||||
// Check directory is empty first though
|
||||
dir := item.node.(*Dir)
|
||||
empty, err := dir.isEmpty()
|
||||
if err != nil {
|
||||
fs.ErrorLog(path, "Dir.Remove dir error: %v", err)
|
||||
return err
|
||||
}
|
||||
if !empty {
|
||||
// return fuse.ENOTEMPTY - doesn't exist though so use EEXIST
|
||||
fs.ErrorLog(path, "Dir.Remove not empty")
|
||||
return fuse.EEXIST
|
||||
}
|
||||
default:
|
||||
fs.ErrorLog(path, "Dir.Remove unknown type %T", item)
|
||||
return errors.Errorf("unknown type %T", item)
|
||||
}
|
||||
// Remove the item from the directory listing
|
||||
d.delObject(req.Name)
|
||||
fs.Debug(path, "Dir.Remove OK")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Check interface satisfied
|
||||
var _ fusefs.NodeRenamer = (*Dir)(nil)
|
||||
|
||||
// Rename the file
|
||||
func (d *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDir fusefs.Node) error {
|
||||
oldPath := path.Join(d.path, req.OldName)
|
||||
destDir, ok := newDir.(*Dir)
|
||||
if !ok {
|
||||
err := errors.Errorf("Unknown Dir type %T", newDir)
|
||||
fs.ErrorLog(oldPath, "Dir.Rename error: %v", err)
|
||||
return err
|
||||
}
|
||||
newPath := path.Join(destDir.path, req.NewName)
|
||||
fs.Debug(oldPath, "Dir.Rename to %q", newPath)
|
||||
oldItem, err := d.lookupNode(req.OldName)
|
||||
if err != nil {
|
||||
fs.ErrorLog(oldPath, "Dir.Rename error: %v", err)
|
||||
return err
|
||||
}
|
||||
var newObj fs.BasicInfo
|
||||
switch x := oldItem.o.(type) {
|
||||
case fs.Object:
|
||||
oldObject := x
|
||||
do, ok := d.f.(fs.Mover)
|
||||
if !ok {
|
||||
err := errors.Errorf("Fs %q can't Move files", d.f)
|
||||
fs.ErrorLog(oldPath, "Dir.Rename error: %v", err)
|
||||
return err
|
||||
}
|
||||
newObject, err := do.Move(oldObject, newPath)
|
||||
if err != nil {
|
||||
fs.ErrorLog(oldPath, "Dir.Rename error: %v", err)
|
||||
return err
|
||||
}
|
||||
newObj = newObject
|
||||
case *fs.Dir:
|
||||
oldDir := oldItem.node.(*Dir)
|
||||
empty, err := oldDir.isEmpty()
|
||||
if err != nil {
|
||||
fs.ErrorLog(oldPath, "Dir.Rename dir error: %v", err)
|
||||
return err
|
||||
}
|
||||
if !empty {
|
||||
// return fuse.ENOTEMPTY - doesn't exist though so use EEXIST
|
||||
fs.ErrorLog(oldPath, "Dir.Rename can't rename non empty directory")
|
||||
return fuse.EEXIST
|
||||
}
|
||||
newObj = &fs.Dir{
|
||||
Name: newPath,
|
||||
When: time.Now(),
|
||||
}
|
||||
default:
|
||||
err = errors.Errorf("unknown type %T", oldItem)
|
||||
fs.ErrorLog(d.path, "Dir.ReadDirAll error: %v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
// Show moved - delete from old dir and add to new
|
||||
d.delObject(req.OldName)
|
||||
destDir.addObject(newObj, nil)
|
||||
|
||||
// FIXME need to flush the dir also
|
||||
|
||||
// FIXME use DirMover to move a directory?
|
||||
// or maybe use MoveDir which can move anything
|
||||
// fallback to Copy/Delete if no Move?
|
||||
// if dir is empty then can move it
|
||||
|
||||
fs.ErrorLog(newPath, "Dir.Rename renamed from %q", oldPath)
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,121 @@
|
|||
// +build linux darwin freebsd
|
||||
|
||||
package mount
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestDirLs(t *testing.T) {
|
||||
run.checkDir(t, "")
|
||||
|
||||
run.mkdir(t, "a directory")
|
||||
run.createFile(t, "a file", "hello")
|
||||
|
||||
run.checkDir(t, "a directory/|a file 5")
|
||||
|
||||
run.rmdir(t, "a directory")
|
||||
run.rm(t, "a file")
|
||||
|
||||
run.checkDir(t, "")
|
||||
}
|
||||
|
||||
func TestDirCreateAndRemoveDir(t *testing.T) {
|
||||
run.mkdir(t, "dir")
|
||||
run.mkdir(t, "dir/subdir")
|
||||
run.checkDir(t, "dir/|dir/subdir/")
|
||||
|
||||
// Check we can't delete a directory with stuff in
|
||||
err := os.Remove(run.path("dir"))
|
||||
assert.Error(t, err, "file exists")
|
||||
|
||||
// Now delete subdir then dir - should produce no errors
|
||||
run.rmdir(t, "dir/subdir")
|
||||
run.checkDir(t, "dir/")
|
||||
run.rmdir(t, "dir")
|
||||
run.checkDir(t, "")
|
||||
}
|
||||
|
||||
func TestDirCreateAndRemoveFile(t *testing.T) {
|
||||
run.mkdir(t, "dir")
|
||||
run.createFile(t, "dir/file", "potato")
|
||||
run.checkDir(t, "dir/|dir/file 6")
|
||||
|
||||
// Check we can't delete a directory with stuff in
|
||||
err := os.Remove(run.path("dir"))
|
||||
assert.Error(t, err, "file exists")
|
||||
|
||||
// Now delete file
|
||||
run.rm(t, "dir/file")
|
||||
|
||||
run.checkDir(t, "dir/")
|
||||
run.rmdir(t, "dir")
|
||||
run.checkDir(t, "")
|
||||
}
|
||||
|
||||
func TestDirRenameFile(t *testing.T) {
|
||||
run.mkdir(t, "dir")
|
||||
run.createFile(t, "file", "potato")
|
||||
run.checkDir(t, "dir/|file 6")
|
||||
|
||||
err := os.Rename(run.path("file"), run.path("dir/file2"))
|
||||
require.NoError(t, err)
|
||||
run.checkDir(t, "dir/|dir/file2 6")
|
||||
|
||||
err = os.Rename(run.path("dir/file2"), run.path("dir/file3"))
|
||||
require.NoError(t, err)
|
||||
run.checkDir(t, "dir/|dir/file3 6")
|
||||
|
||||
run.rm(t, "dir/file3")
|
||||
run.rmdir(t, "dir")
|
||||
run.checkDir(t, "")
|
||||
}
|
||||
|
||||
func TestDirRenameEmptyDir(t *testing.T) {
|
||||
run.mkdir(t, "dir")
|
||||
run.mkdir(t, "dir1")
|
||||
run.checkDir(t, "dir/|dir1/")
|
||||
|
||||
err := os.Rename(run.path("dir1"), run.path("dir/dir2"))
|
||||
require.NoError(t, err)
|
||||
run.checkDir(t, "dir/|dir/dir2/")
|
||||
|
||||
err = os.Rename(run.path("dir/dir2"), run.path("dir/dir3"))
|
||||
require.NoError(t, err)
|
||||
run.checkDir(t, "dir/|dir/dir3/")
|
||||
|
||||
run.rmdir(t, "dir/dir3")
|
||||
run.rmdir(t, "dir")
|
||||
run.checkDir(t, "")
|
||||
}
|
||||
|
||||
func TestDirRenameFullDir(t *testing.T) {
|
||||
run.mkdir(t, "dir")
|
||||
run.mkdir(t, "dir1")
|
||||
run.createFile(t, "dir1/potato.txt", "maris piper")
|
||||
run.checkDir(t, "dir/|dir1/|dir1/potato.txt 11")
|
||||
|
||||
err := os.Rename(run.path("dir1"), run.path("dir/dir2"))
|
||||
require.Error(t, err, "file exists")
|
||||
// Can't currently rename directories with stuff in
|
||||
/*
|
||||
require.NoError(t, err)
|
||||
run.checkDir(t, "dir/|dir/dir2/|dir/dir2/potato.txt 11")
|
||||
|
||||
err = os.Rename(run.path("dir/dir2"), run.path("dir/dir3"))
|
||||
require.NoError(t, err)
|
||||
run.checkDir(t, "dir/|dir/dir3/|dir/dir3/potato.txt 11")
|
||||
|
||||
run.rm(t, "dir/dir3/potato.txt")
|
||||
run.rmdir(t, "dir/dir3")
|
||||
*/
|
||||
|
||||
run.rm(t, "dir1/potato.txt")
|
||||
run.rmdir(t, "dir1")
|
||||
run.rmdir(t, "dir")
|
||||
run.checkDir(t, "")
|
||||
}
|
||||
|
|
@ -0,0 +1,142 @@
|
|||
// +build linux darwin freebsd
|
||||
|
||||
package mount
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"bazil.org/fuse"
|
||||
fusefs "bazil.org/fuse/fs"
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/pkg/errors"
|
||||
"golang.org/x/net/context"
|
||||
)
|
||||
|
||||
// File represents a file
|
||||
type File struct {
|
||||
d *Dir // parent directory - read only
|
||||
size int64 // size of file - read and written with atomic
|
||||
mu sync.RWMutex // protects the following
|
||||
o fs.Object // NB o may be nil if file is being written
|
||||
writers int // number of writers for this file
|
||||
}
|
||||
|
||||
// newFile creates a new File
|
||||
func newFile(d *Dir, o fs.Object) *File {
|
||||
return &File{
|
||||
d: d,
|
||||
o: o,
|
||||
}
|
||||
}
|
||||
|
||||
// addWriters increments or decrements the writers
|
||||
func (f *File) addWriters(n int) {
|
||||
f.mu.Lock()
|
||||
f.writers += n
|
||||
f.mu.Unlock()
|
||||
}
|
||||
|
||||
// Check interface satisfied
|
||||
var _ fusefs.Node = (*File)(nil)
|
||||
|
||||
// Attr fills out the attributes for the file
|
||||
func (f *File) Attr(ctx context.Context, a *fuse.Attr) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
fs.Debug(f.o, "File.Attr")
|
||||
a.Mode = filePerms
|
||||
// if o is nil it isn't valid yet, so return the size so far
|
||||
if f.o == nil {
|
||||
a.Size = uint64(atomic.LoadInt64(&f.size))
|
||||
} else {
|
||||
a.Size = uint64(f.o.Size())
|
||||
if !noModTime {
|
||||
modTime := f.o.ModTime()
|
||||
a.Atime = modTime
|
||||
a.Mtime = modTime
|
||||
a.Ctime = modTime
|
||||
a.Crtime = modTime
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Update the size while writing
|
||||
func (f *File) written(n int64) {
|
||||
atomic.AddInt64(&f.size, n)
|
||||
}
|
||||
|
||||
// Update the object when written
|
||||
func (f *File) setObject(o fs.Object) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.o = o
|
||||
f.d.addObject(o, f)
|
||||
}
|
||||
|
||||
// Wait for f.o to become non nil for a short time returning it or an
|
||||
// error
|
||||
//
|
||||
// Call without the mutex held
|
||||
func (f *File) waitForValidObject() (o fs.Object, err error) {
|
||||
for i := 0; i < 50; i++ {
|
||||
f.mu.Lock()
|
||||
o = f.o
|
||||
writers := f.writers
|
||||
f.mu.Unlock()
|
||||
if o != nil {
|
||||
return o, nil
|
||||
}
|
||||
if writers == 0 {
|
||||
return nil, errors.New("can't open file - writer failed")
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
return nil, fuse.ENOENT
|
||||
}
|
||||
|
||||
// Check interface satisfied
|
||||
var _ fusefs.NodeOpener = (*File)(nil)
|
||||
|
||||
// Open the file for read or write
|
||||
func (f *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fusefs.Handle, error) {
|
||||
// if o is nil it isn't valid yet
|
||||
o, err := f.waitForValidObject()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
fs.Debug(o, "File.Open")
|
||||
|
||||
// Files aren't seekable
|
||||
resp.Flags |= fuse.OpenNonSeekable
|
||||
|
||||
switch {
|
||||
case req.Flags.IsReadOnly():
|
||||
return newReadFileHandle(o)
|
||||
case req.Flags.IsWriteOnly():
|
||||
src := newCreateInfo(f.d.f, o.Remote())
|
||||
fh, err := newWriteFileHandle(f.d, f, src)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return fh, nil
|
||||
case req.Flags.IsReadWrite():
|
||||
return nil, errors.New("can't open read and write")
|
||||
}
|
||||
|
||||
/*
|
||||
// File was opened in append-only mode, all writes will go to end
|
||||
// of file. OS X does not provide this information.
|
||||
OpenAppend OpenFlags = syscall.O_APPEND
|
||||
OpenCreate OpenFlags = syscall.O_CREAT
|
||||
OpenDirectory OpenFlags = syscall.O_DIRECTORY
|
||||
OpenExclusive OpenFlags = syscall.O_EXCL
|
||||
OpenNonblock OpenFlags = syscall.O_NONBLOCK
|
||||
OpenSync OpenFlags = syscall.O_SYNC
|
||||
OpenTruncate OpenFlags = syscall.O_TRUNC
|
||||
*/
|
||||
return nil, errors.New("can't figure out how to open")
|
||||
}
|
||||
|
|
@ -0,0 +1,67 @@
|
|||
// FUSE main Fs
|
||||
|
||||
// +build linux darwin freebsd
|
||||
|
||||
package mount
|
||||
|
||||
import (
|
||||
"bazil.org/fuse"
|
||||
fusefs "bazil.org/fuse/fs"
|
||||
"github.com/ncw/rclone/fs"
|
||||
)
|
||||
|
||||
// Default permissions
|
||||
const (
|
||||
dirPerms = 0755
|
||||
filePerms = 0644
|
||||
)
|
||||
|
||||
// FS represents the top level filing system
|
||||
type FS struct {
|
||||
f fs.Fs
|
||||
}
|
||||
|
||||
// Check interface satistfied
|
||||
var _ fusefs.FS = (*FS)(nil)
|
||||
|
||||
// Root returns the root node
|
||||
func (f *FS) Root() (fusefs.Node, error) {
|
||||
fs.Debug(f.f, "Root()")
|
||||
return newDir(f.f, ""), nil
|
||||
}
|
||||
|
||||
// mount the file system
|
||||
//
|
||||
// The mount point will be ready when this returns.
|
||||
//
|
||||
// returns an error, and an error channel for the serve process to
|
||||
// report an error when fusermount is called.
|
||||
func mount(f fs.Fs, mountpoint string) (<-chan error, error) {
|
||||
c, err := fuse.Mount(mountpoint)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
filesys := &FS{
|
||||
f: f,
|
||||
}
|
||||
|
||||
// Serve the mount point in the background returning error to errChan
|
||||
errChan := make(chan error, 1)
|
||||
go func() {
|
||||
err := fusefs.Serve(c, filesys)
|
||||
closeErr := c.Close()
|
||||
if err == nil {
|
||||
err = closeErr
|
||||
}
|
||||
errChan <- err
|
||||
}()
|
||||
|
||||
// check if the mount process has an error to report
|
||||
<-c.Ready
|
||||
if err := c.MountError; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return errChan, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,248 @@
|
|||
// +build linux darwin freebsd
|
||||
|
||||
// Test suite for rclonefs
|
||||
|
||||
package mount
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/ncw/rclone/fs"
|
||||
_ "github.com/ncw/rclone/fs/all"
|
||||
"github.com/ncw/rclone/fstest"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// Globals
|
||||
var (
|
||||
RemoteName = flag.String("remote", "", "Remote to test with, defaults to local filesystem")
|
||||
SubDir = flag.Bool("subdir", false, "Set to test with a sub directory")
|
||||
Verbose = flag.Bool("verbose", false, "Set to enable logging")
|
||||
DumpHeaders = flag.Bool("dump-headers", false, "Set to dump headers (needs -verbose)")
|
||||
DumpBodies = flag.Bool("dump-bodies", false, "Set to dump bodies (needs -verbose)")
|
||||
Individual = flag.Bool("individual", false, "Make individual bucket/container/directory for each test - much slower")
|
||||
LowLevelRetries = flag.Int("low-level-retries", 10, "Number of low level retries")
|
||||
)
|
||||
|
||||
// TestMain drives the tests
|
||||
func TestMain(m *testing.M) {
|
||||
flag.Parse()
|
||||
run = newRun()
|
||||
rc := m.Run()
|
||||
run.Finalise()
|
||||
os.Exit(rc)
|
||||
}
|
||||
|
||||
// Run holds the remotes for a test run
|
||||
type Run struct {
|
||||
mountPath string
|
||||
fremote fs.Fs
|
||||
fremoteName string
|
||||
cleanRemote func()
|
||||
umountResult <-chan error
|
||||
}
|
||||
|
||||
// run holds the master Run data
|
||||
var run *Run
|
||||
|
||||
// newRun initialise the remote mount for testing and returns a run
|
||||
// object.
|
||||
//
|
||||
// r.fremote is an empty remote Fs
|
||||
//
|
||||
// Finalise() will tidy them away when done.
|
||||
func newRun() *Run {
|
||||
r := &Run{
|
||||
umountResult: make(chan error, 1),
|
||||
}
|
||||
|
||||
// Never ask for passwords, fail instead.
|
||||
// If your local config is encrypted set environment variable
|
||||
// "RCLONE_CONFIG_PASS=hunter2" (or your password)
|
||||
*fs.AskPassword = false
|
||||
fs.LoadConfig()
|
||||
fs.Config.Verbose = *Verbose
|
||||
fs.Config.Quiet = !*Verbose
|
||||
fs.Config.DumpHeaders = *DumpHeaders
|
||||
fs.Config.DumpBodies = *DumpBodies
|
||||
fs.Config.LowLevelRetries = *LowLevelRetries
|
||||
var err error
|
||||
r.fremote, r.fremoteName, r.cleanRemote, err = fstest.RandomRemote(*RemoteName, *SubDir)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to open remote %q: %v", *RemoteName, err)
|
||||
}
|
||||
|
||||
r.mountPath, err = ioutil.TempDir("", "rclonefs-mount")
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to create mount dir: %v", err)
|
||||
}
|
||||
|
||||
// Mount it up
|
||||
r.mount()
|
||||
|
||||
return r
|
||||
}
|
||||
|
||||
func (r *Run) mount() {
|
||||
log.Printf("mount %q %q", r.fremote, r.mountPath)
|
||||
var err error
|
||||
r.umountResult, err = mount(r.fremote, r.mountPath)
|
||||
if err != nil {
|
||||
log.Fatalf("umount failed: %v", err)
|
||||
}
|
||||
log.Printf("mount OK")
|
||||
}
|
||||
|
||||
func (r *Run) umount() {
|
||||
log.Printf("Calling fusermount -u %q", r.mountPath)
|
||||
err := exec.Command("fusermount", "-u", r.mountPath).Run()
|
||||
if err != nil {
|
||||
log.Printf("fusermount failed: %v", err)
|
||||
}
|
||||
log.Printf("Waiting for umount")
|
||||
err = <-r.umountResult
|
||||
if err != nil {
|
||||
log.Fatalf("umount failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Finalise cleans the remote and unmounts
|
||||
func (r *Run) Finalise() {
|
||||
r.umount()
|
||||
r.cleanRemote()
|
||||
err := os.RemoveAll(r.mountPath)
|
||||
if err != nil {
|
||||
log.Printf("Failed to clean mountPath %q: %v", r.mountPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Run) path(filepath string) string {
|
||||
return path.Join(run.mountPath, filepath)
|
||||
}
|
||||
|
||||
type dirMap map[string]struct{}
|
||||
|
||||
// Create a dirMap from a string
|
||||
func newDirMap(dirString string) (dm dirMap) {
|
||||
dm = make(dirMap)
|
||||
for _, entry := range strings.Split(dirString, "|") {
|
||||
if entry != "" {
|
||||
dm[entry] = struct{}{}
|
||||
}
|
||||
}
|
||||
return dm
|
||||
}
|
||||
|
||||
// Returns a dirmap with only the files in
|
||||
func (dm dirMap) filesOnly() dirMap {
|
||||
newDm := make(dirMap)
|
||||
for name := range dm {
|
||||
if !strings.HasSuffix(name, "/") {
|
||||
newDm[name] = struct{}{}
|
||||
}
|
||||
}
|
||||
return newDm
|
||||
}
|
||||
|
||||
// reads the local tree into dir
|
||||
func (r *Run) readLocal(t *testing.T, dir dirMap, filepath string) {
|
||||
realPath := r.path(filepath)
|
||||
files, err := ioutil.ReadDir(realPath)
|
||||
require.NoError(t, err)
|
||||
for _, fi := range files {
|
||||
name := path.Join(filepath, fi.Name())
|
||||
if fi.IsDir() {
|
||||
dir[name+"/"] = struct{}{}
|
||||
r.readLocal(t, dir, name)
|
||||
assert.Equal(t, fi.Mode().Perm(), os.FileMode(dirPerms))
|
||||
} else {
|
||||
dir[fmt.Sprintf("%s %d", name, fi.Size())] = struct{}{}
|
||||
assert.Equal(t, fi.Mode().Perm(), os.FileMode(filePerms))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// reads the remote tree into dir
|
||||
func (r *Run) readRemote(t *testing.T, dir dirMap, filepath string) {
|
||||
objs, dirs, err := fs.NewLister().SetLevel(1).Start(r.fremote, filepath).GetAll()
|
||||
if err == fs.ErrorDirNotFound {
|
||||
return
|
||||
}
|
||||
require.NoError(t, err)
|
||||
for _, obj := range objs {
|
||||
dir[fmt.Sprintf("%s %d", obj.Remote(), obj.Size())] = struct{}{}
|
||||
}
|
||||
for _, d := range dirs {
|
||||
name := d.Remote()
|
||||
dir[name+"/"] = struct{}{}
|
||||
r.readRemote(t, dir, name)
|
||||
}
|
||||
}
|
||||
|
||||
// checkDir checks the local and remote against the string passed in
|
||||
func (r *Run) checkDir(t *testing.T, dirString string) {
|
||||
dm := newDirMap(dirString)
|
||||
localDm := make(dirMap)
|
||||
r.readLocal(t, localDm, "")
|
||||
remoteDm := make(dirMap)
|
||||
r.readRemote(t, remoteDm, "")
|
||||
// Ignore directories for remote compare
|
||||
assert.Equal(t, dm.filesOnly(), remoteDm.filesOnly(), "expected vs remote")
|
||||
assert.Equal(t, dm, localDm, "expected vs fuse mount")
|
||||
}
|
||||
|
||||
func (r *Run) createFile(t *testing.T, filepath string, contents string) {
|
||||
filepath = r.path(filepath)
|
||||
err := ioutil.WriteFile(filepath, []byte(contents), 0600)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func (r *Run) readFile(t *testing.T, filepath string) string {
|
||||
filepath = r.path(filepath)
|
||||
result, err := ioutil.ReadFile(filepath)
|
||||
require.NoError(t, err)
|
||||
return string(result)
|
||||
}
|
||||
|
||||
func (r *Run) mkdir(t *testing.T, filepath string) {
|
||||
filepath = r.path(filepath)
|
||||
err := os.Mkdir(filepath, 0700)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func (r *Run) rm(t *testing.T, filepath string) {
|
||||
filepath = r.path(filepath)
|
||||
err := os.Remove(filepath)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func (r *Run) rmdir(t *testing.T, filepath string) {
|
||||
filepath = r.path(filepath)
|
||||
err := os.Remove(filepath)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// Check that the Fs is mounted by seeing if the mountpoint is
|
||||
// in the mount output
|
||||
func TestMount(t *testing.T) {
|
||||
out, err := exec.Command("mount").Output()
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, string(out), run.mountPath)
|
||||
}
|
||||
|
||||
// Check root directory is present and correct
|
||||
func TestRoot(t *testing.T) {
|
||||
fi, err := os.Lstat(run.mountPath)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, fi.IsDir())
|
||||
assert.Equal(t, fi.Mode().Perm(), os.FileMode(dirPerms))
|
||||
}
|
||||
|
|
@ -0,0 +1,117 @@
|
|||
// Package mount implents a FUSE mounting system for rclone remotes.
|
||||
|
||||
// +build linux darwin freebsd
|
||||
|
||||
package mount
|
||||
|
||||
import (
|
||||
"bazil.org/fuse"
|
||||
"github.com/ncw/rclone/cmd"
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
// Globals
|
||||
var (
|
||||
noModTime = false
|
||||
debugFUSE = false
|
||||
)
|
||||
|
||||
func init() {
|
||||
cmd.Root.AddCommand(mountCmd)
|
||||
mountCmd.Flags().BoolVarP(&noModTime, "no-modtime", "", false, "Don't read the modification time (can speed things up).")
|
||||
mountCmd.Flags().BoolVarP(&debugFUSE, "debug-fuse", "", false, "Debug the FUSE internals - needs -v.")
|
||||
}
|
||||
|
||||
var mountCmd = &cobra.Command{
|
||||
Use: "mount remote:path /path/to/mountpoint",
|
||||
Short: `Mount the remote as a mountpoint. **EXPERIMENTAL**`,
|
||||
Long: `
|
||||
rclone mount allows Linux and macOS to mount any of Rclone's cloud storage
|
||||
systems as a file system with FUSE.
|
||||
|
||||
This is **EXPERIMENTAL** - use with care.
|
||||
|
||||
First set up your remote using ` + "`rclone config`" + `. Check it works with ` + "`rclone ls`" + ` etc.
|
||||
|
||||
Start the mount like this
|
||||
|
||||
rclone mount remote:path/to/files /path/to/local/mount &
|
||||
|
||||
Stop the mount with
|
||||
|
||||
fusermount -u /path/to/local/mount
|
||||
|
||||
Or with OS X
|
||||
|
||||
umount -u /path/to/local/mount
|
||||
|
||||
### Limitations ###
|
||||
|
||||
This can only read files seqentially, or write files sequentially. It
|
||||
can't read and write or seek in files.
|
||||
|
||||
rclonefs inherits rclone's directory handling. In rclone's world
|
||||
directories don't really exist. This means that empty directories
|
||||
will have a tendency to disappear once they fall out of the directory
|
||||
cache.
|
||||
|
||||
The bucket based FSes (eg swift, s3, google compute storage, b2) won't
|
||||
work from the root - you will need to specify a bucket, or a path
|
||||
within the bucket. So ` + "`swift:`" + ` won't work whereas ` + "`swift:bucket`" + ` will
|
||||
as will ` + "`swift:bucket/path`" + `.
|
||||
|
||||
### rclone mount vs rclone sync/copy ##
|
||||
|
||||
File systems expect things to be 100% reliable, whereas cloud storage
|
||||
systems are a long way from 100% reliable. The rclone sync/copy
|
||||
commands cope with this with lots of retries. However rclone mount
|
||||
can't use retries in the same way without making local copies of the
|
||||
uploads. This might happen in the future, but for the moment rclone
|
||||
mount won't do that, so will be less reliable than the rclone command.
|
||||
|
||||
### Bugs ###
|
||||
|
||||
* All the remotes should work for read, but some may not for write
|
||||
* those which need to know the size in advance won't - eg B2
|
||||
* maybe should pass in size as -1 to mean work it out
|
||||
|
||||
### TODO ###
|
||||
|
||||
* Tests
|
||||
* Check hashes on upload/download
|
||||
* Preserve timestamps
|
||||
* Move directories
|
||||
`,
|
||||
RunE: func(command *cobra.Command, args []string) error {
|
||||
cmd.CheckArgs(2, 2, command, args)
|
||||
fdst := cmd.NewFsDst(args)
|
||||
return Mount(fdst, args[1])
|
||||
},
|
||||
}
|
||||
|
||||
// Mount mounts the remote at mountpoint.
|
||||
//
|
||||
// If noModTime is set then it
|
||||
func Mount(f fs.Fs, mountpoint string) error {
|
||||
if debugFUSE {
|
||||
fuse.Debug = func(msg interface{}) {
|
||||
fs.Debug("fuse", "%v", msg)
|
||||
}
|
||||
}
|
||||
|
||||
// Mount it
|
||||
errChan, err := mount(f, mountpoint)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to mount FUSE fs")
|
||||
}
|
||||
|
||||
// Wait for umount
|
||||
err = <-errChan
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to umount FUSE fs")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
// Build for mount for unsupported platforms to stop go complaining
|
||||
// about "no buildable Go source files "
|
||||
|
||||
// +build !linux,!darwin,!freebsd
|
||||
|
||||
package mount
|
||||
|
|
@ -0,0 +1,130 @@
|
|||
// +build linux darwin freebsd
|
||||
|
||||
package mount
|
||||
|
||||
import (
|
||||
"io"
|
||||
"sync"
|
||||
|
||||
"bazil.org/fuse"
|
||||
fusefs "bazil.org/fuse/fs"
|
||||
"github.com/ncw/rclone/fs"
|
||||
"golang.org/x/net/context"
|
||||
)
|
||||
|
||||
// ReadFileHandle is an open for read file handle on a File
|
||||
type ReadFileHandle struct {
|
||||
mu sync.Mutex
|
||||
closed bool // set if handle has been closed
|
||||
r io.ReadCloser
|
||||
o fs.Object
|
||||
readCalled bool // set if read has been called
|
||||
}
|
||||
|
||||
func newReadFileHandle(o fs.Object) (*ReadFileHandle, error) {
|
||||
r, err := o.Open()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &ReadFileHandle{
|
||||
r: r,
|
||||
o: o,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Check interface satisfied
|
||||
var _ fusefs.Handle = (*ReadFileHandle)(nil)
|
||||
|
||||
// Check interface satisfied
|
||||
var _ fusefs.HandleReader = (*ReadFileHandle)(nil)
|
||||
|
||||
// Read from the file handle
|
||||
func (fh *ReadFileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error {
|
||||
fs.Debug(fh.o, "ReadFileHandle.Open")
|
||||
if fh.closed {
|
||||
fs.ErrorLog(fh.o, "ReadFileHandle.Read error: %v", errClosedFileHandle)
|
||||
return errClosedFileHandle
|
||||
}
|
||||
fh.readCalled = true
|
||||
// We don't actually enforce Offset to match where previous read
|
||||
// ended. Maybe we should, but that would mean'd we need to track
|
||||
// it. The kernel *should* do it for us, based on the
|
||||
// fuse.OpenNonSeekable flag.
|
||||
//
|
||||
// One exception to the above is if we fail to fully populate a
|
||||
// page cache page; a read into page cache is always page aligned.
|
||||
// Make sure we never serve a partial read, to avoid that.
|
||||
buf := make([]byte, req.Size)
|
||||
n, err := io.ReadFull(fh.r, buf)
|
||||
if err == io.ErrUnexpectedEOF || err == io.EOF {
|
||||
err = nil
|
||||
}
|
||||
resp.Data = buf[:n]
|
||||
if err != nil {
|
||||
fs.ErrorLog(fh.o, "ReadFileHandle.Open error: %v", err)
|
||||
} else {
|
||||
fs.Debug(fh.o, "ReadFileHandle.Open OK")
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// close the file handle returning errClosedFileHandle if it has been
|
||||
// closed already.
|
||||
//
|
||||
// Must be called with fh.mu held
|
||||
func (fh *ReadFileHandle) close() error {
|
||||
if fh.closed {
|
||||
return errClosedFileHandle
|
||||
}
|
||||
fh.closed = true
|
||||
return fh.r.Close()
|
||||
}
|
||||
|
||||
// Check interface satisfied
|
||||
var _ fusefs.HandleFlusher = (*ReadFileHandle)(nil)
|
||||
|
||||
// Flush is called each time the file or directory is closed.
|
||||
// Because there can be multiple file descriptors referring to a
|
||||
// single opened file, Flush can be called multiple times.
|
||||
func (fh *ReadFileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error {
|
||||
fh.mu.Lock()
|
||||
defer fh.mu.Unlock()
|
||||
fs.Debug(fh.o, "ReadFileHandle.Flush")
|
||||
// If Read hasn't been called then ignore the Flush - Release
|
||||
// will pick it up
|
||||
if !fh.readCalled {
|
||||
fs.Debug(fh.o, "ReadFileHandle.Flush ignoring flush on unread handle")
|
||||
return nil
|
||||
|
||||
}
|
||||
err := fh.close()
|
||||
if err != nil {
|
||||
fs.ErrorLog(fh.o, "ReadFileHandle.Flush error: %v", err)
|
||||
return err
|
||||
}
|
||||
fs.Debug(fh.o, "ReadFileHandle.Flush OK")
|
||||
return nil
|
||||
}
|
||||
|
||||
var _ fusefs.HandleReleaser = (*ReadFileHandle)(nil)
|
||||
|
||||
// Release is called when we are finished with the file handle
|
||||
//
|
||||
// It isn't called directly from userspace so the error is ignored by
|
||||
// the kernel
|
||||
func (fh *ReadFileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) error {
|
||||
fh.mu.Lock()
|
||||
defer fh.mu.Unlock()
|
||||
if fh.closed {
|
||||
fs.Debug(fh.o, "ReadFileHandle.Release nothing to do")
|
||||
return nil
|
||||
}
|
||||
fs.Debug(fh.o, "ReadFileHandle.Release closing")
|
||||
err := fh.close()
|
||||
if err != nil {
|
||||
fs.ErrorLog(fh.o, "ReadFileHandle.Release error: %v", err)
|
||||
} else {
|
||||
fs.Debug(fh.o, "ReadFileHandle.Release OK")
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
|
@ -0,0 +1,75 @@
|
|||
// +build linux darwin freebsd
|
||||
|
||||
package mount
|
||||
|
||||
import (
|
||||
"io"
|
||||
"os"
|
||||
"syscall"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
// Read by byte including don't read any bytes
|
||||
func TestReadByByte(t *testing.T) {
|
||||
var data = []byte("hellohello")
|
||||
run.createFile(t, "testfile", string(data))
|
||||
run.checkDir(t, "testfile 10")
|
||||
|
||||
for i := 0; i < len(data); i++ {
|
||||
fd, err := os.Open(run.path("testfile"))
|
||||
assert.NoError(t, err)
|
||||
for j := 0; j < i; j++ {
|
||||
buf := make([]byte, 1)
|
||||
n, err := io.ReadFull(fd, buf)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 1, n)
|
||||
assert.Equal(t, buf[0], data[j])
|
||||
}
|
||||
err = fd.Close()
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
run.rm(t, "testfile")
|
||||
}
|
||||
|
||||
// Test double close
|
||||
func TestReadFileDoubleClose(t *testing.T) {
|
||||
run.createFile(t, "testdoubleclose", "hello")
|
||||
|
||||
in, err := os.Open(run.path("testdoubleclose"))
|
||||
assert.NoError(t, err)
|
||||
fd := in.Fd()
|
||||
|
||||
fd1, err := syscall.Dup(int(fd))
|
||||
assert.NoError(t, err)
|
||||
|
||||
fd2, err := syscall.Dup(int(fd))
|
||||
assert.NoError(t, err)
|
||||
|
||||
// close one of the dups - should produce no error
|
||||
err = syscall.Close(fd1)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// read from the file
|
||||
buf := make([]byte, 1)
|
||||
_, err = in.Read(buf)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// close it
|
||||
err = in.Close()
|
||||
assert.NoError(t, err)
|
||||
|
||||
// read from the other dup - should produce no error as this
|
||||
// file is now buffered
|
||||
n, err := syscall.Read(fd2, buf)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 1, n)
|
||||
|
||||
// close the dup - should produce an error
|
||||
err = syscall.Close(fd2)
|
||||
assert.Error(t, err, "input/output error")
|
||||
|
||||
run.rm(t, "testdoubleclose")
|
||||
}
|
||||
|
|
@ -0,0 +1,157 @@
|
|||
// +build linux darwin freebsd
|
||||
|
||||
package mount
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"sync"
|
||||
|
||||
"bazil.org/fuse"
|
||||
fusefs "bazil.org/fuse/fs"
|
||||
"github.com/ncw/rclone/fs"
|
||||
"golang.org/x/net/context"
|
||||
)
|
||||
|
||||
var errClosedFileHandle = errors.New("Attempt to use closed file handle")
|
||||
|
||||
// WriteFileHandle is an open for write handle on a File
|
||||
type WriteFileHandle struct {
|
||||
mu sync.Mutex
|
||||
closed bool // set if handle has been closed
|
||||
remote string
|
||||
pipeReader *io.PipeReader
|
||||
pipeWriter *io.PipeWriter
|
||||
o fs.Object
|
||||
result chan error
|
||||
file *File
|
||||
writeCalled bool // set the first time Write() is called
|
||||
}
|
||||
|
||||
// Check interface satisfied
|
||||
var _ fusefs.Handle = (*WriteFileHandle)(nil)
|
||||
|
||||
func newWriteFileHandle(d *Dir, f *File, src fs.ObjectInfo) (*WriteFileHandle, error) {
|
||||
fh := &WriteFileHandle{
|
||||
remote: src.Remote(),
|
||||
result: make(chan error, 1),
|
||||
file: f,
|
||||
}
|
||||
fh.pipeReader, fh.pipeWriter = io.Pipe()
|
||||
go func() {
|
||||
o, err := d.f.Put(fh.pipeReader, src)
|
||||
fh.o = o
|
||||
fh.result <- err
|
||||
}()
|
||||
fh.file.addWriters(1)
|
||||
return fh, nil
|
||||
}
|
||||
|
||||
// Check interface satisfied
|
||||
var _ fusefs.HandleWriter = (*WriteFileHandle)(nil)
|
||||
|
||||
// Write data to the file handle
|
||||
func (fh *WriteFileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse.WriteResponse) error {
|
||||
fs.Debug(fh.remote, "WriteFileHandle.Write len=%d", len(req.Data))
|
||||
fh.mu.Lock()
|
||||
defer fh.mu.Unlock()
|
||||
if fh.closed {
|
||||
fs.ErrorLog(fh.remote, "WriteFileHandle.Write error: %v", errClosedFileHandle)
|
||||
return errClosedFileHandle
|
||||
}
|
||||
fh.writeCalled = true
|
||||
// FIXME should probably check the file isn't being seeked?
|
||||
n, err := fh.pipeWriter.Write(req.Data)
|
||||
resp.Size = n
|
||||
fh.file.written(int64(n))
|
||||
if err != nil {
|
||||
fs.ErrorLog(fh.remote, "WriteFileHandle.Write error: %v", err)
|
||||
return err
|
||||
}
|
||||
fs.Debug(fh.remote, "WriteFileHandle.Write OK (%d bytes written)", n)
|
||||
return nil
|
||||
}
|
||||
|
||||
// close the file handle returning errClosedFileHandle if it has been
|
||||
// closed already.
|
||||
//
|
||||
// Must be called with fh.mu held
|
||||
func (fh *WriteFileHandle) close() error {
|
||||
if fh.closed {
|
||||
return errClosedFileHandle
|
||||
}
|
||||
fh.closed = true
|
||||
fh.file.addWriters(-1)
|
||||
writeCloseErr := fh.pipeWriter.Close()
|
||||
err := <-fh.result
|
||||
readCloseErr := fh.pipeReader.Close()
|
||||
if err == nil {
|
||||
fh.file.setObject(fh.o)
|
||||
err = writeCloseErr
|
||||
}
|
||||
if err == nil {
|
||||
err = readCloseErr
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Check interface satisfied
|
||||
var _ fusefs.HandleFlusher = (*WriteFileHandle)(nil)
|
||||
|
||||
// Flush is called on each close() of a file descriptor. So if a
|
||||
// filesystem wants to return write errors in close() and the file has
|
||||
// cached dirty data, this is a good place to write back data and
|
||||
// return any errors. Since many applications ignore close() errors
|
||||
// this is not always useful.
|
||||
//
|
||||
// NOTE: The flush() method may be called more than once for each
|
||||
// open(). This happens if more than one file descriptor refers to an
|
||||
// opened file due to dup(), dup2() or fork() calls. It is not
|
||||
// possible to determine if a flush is final, so each flush should be
|
||||
// treated equally. Multiple write-flush sequences are relatively
|
||||
// rare, so this shouldn't be a problem.
|
||||
//
|
||||
// Filesystems shouldn't assume that flush will always be called after
|
||||
// some writes, or that if will be called at all.
|
||||
func (fh *WriteFileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error {
|
||||
fh.mu.Lock()
|
||||
defer fh.mu.Unlock()
|
||||
fs.Debug(fh.remote, "WriteFileHandle.Flush")
|
||||
// If Write hasn't been called then ignore the Flush - Release
|
||||
// will pick it up
|
||||
if !fh.writeCalled {
|
||||
fs.Debug(fh.remote, "WriteFileHandle.Flush ignoring flush on unwritten handle")
|
||||
return nil
|
||||
|
||||
}
|
||||
err := fh.close()
|
||||
if err != nil {
|
||||
fs.ErrorLog(fh.remote, "WriteFileHandle.Flush error: %v", err)
|
||||
} else {
|
||||
fs.Debug(fh.remote, "WriteFileHandle.Flush OK")
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
var _ fusefs.HandleReleaser = (*WriteFileHandle)(nil)
|
||||
|
||||
// Release is called when we are finished with the file handle
|
||||
//
|
||||
// It isn't called directly from userspace so the error is ignored by
|
||||
// the kernel
|
||||
func (fh *WriteFileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) error {
|
||||
fh.mu.Lock()
|
||||
defer fh.mu.Unlock()
|
||||
if fh.closed {
|
||||
fs.Debug(fh.remote, "WriteFileHandle.Release nothing to do")
|
||||
return nil
|
||||
}
|
||||
fs.Debug(fh.remote, "WriteFileHandle.Release closing")
|
||||
err := fh.close()
|
||||
if err != nil {
|
||||
fs.ErrorLog(fh.remote, "WriteFileHandle.Release error: %v", err)
|
||||
} else {
|
||||
fs.Debug(fh.remote, "WriteFileHandle.Release OK")
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
|
@ -0,0 +1,93 @@
|
|||
// +build linux darwin freebsd
|
||||
|
||||
package mount
|
||||
|
||||
import (
|
||||
"os"
|
||||
"syscall"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
// Test writing a file with no write()'s to it
|
||||
func TestWriteFileNoWrite(t *testing.T) {
|
||||
fd, err := os.Create(run.path("testnowrite"))
|
||||
assert.NoError(t, err)
|
||||
|
||||
err = fd.Close()
|
||||
assert.NoError(t, err)
|
||||
|
||||
run.checkDir(t, "testnowrite 0")
|
||||
|
||||
run.rm(t, "testnowrite")
|
||||
}
|
||||
|
||||
// Test open file in directory listing
|
||||
func FIXMETestWriteOpenFileInDirListing(t *testing.T) {
|
||||
fd, err := os.Create(run.path("testnowrite"))
|
||||
assert.NoError(t, err)
|
||||
|
||||
run.checkDir(t, "testnowrite 0")
|
||||
|
||||
err = fd.Close()
|
||||
assert.NoError(t, err)
|
||||
|
||||
run.rm(t, "testnowrite")
|
||||
}
|
||||
|
||||
// Test writing a file and reading it back
|
||||
func TestWriteFileWrite(t *testing.T) {
|
||||
run.createFile(t, "testwrite", "data")
|
||||
run.checkDir(t, "testwrite 4")
|
||||
contents := run.readFile(t, "testwrite")
|
||||
assert.Equal(t, "data", contents)
|
||||
run.rm(t, "testwrite")
|
||||
}
|
||||
|
||||
// Test overwriting a file
|
||||
func TestWriteFileOverwrite(t *testing.T) {
|
||||
run.createFile(t, "testwrite", "data")
|
||||
run.checkDir(t, "testwrite 4")
|
||||
run.createFile(t, "testwrite", "potato")
|
||||
contents := run.readFile(t, "testwrite")
|
||||
assert.Equal(t, "potato", contents)
|
||||
run.rm(t, "testwrite")
|
||||
}
|
||||
|
||||
// Test double close
|
||||
func TestWriteFileDoubleClose(t *testing.T) {
|
||||
out, err := os.Create(run.path("testdoubleclose"))
|
||||
assert.NoError(t, err)
|
||||
fd := out.Fd()
|
||||
|
||||
fd1, err := syscall.Dup(int(fd))
|
||||
assert.NoError(t, err)
|
||||
|
||||
fd2, err := syscall.Dup(int(fd))
|
||||
assert.NoError(t, err)
|
||||
|
||||
// close one of the dups - should produce no error
|
||||
err = syscall.Close(fd1)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// write to the file
|
||||
buf := []byte("hello")
|
||||
n, err := out.Write(buf)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 5, n)
|
||||
|
||||
// close it
|
||||
err = out.Close()
|
||||
assert.NoError(t, err)
|
||||
|
||||
// write to the other dup - should produce an error
|
||||
n, err = syscall.Write(fd2, buf)
|
||||
assert.Error(t, err, "input/output error")
|
||||
|
||||
// close the dup - should produce an error
|
||||
err = syscall.Close(fd2)
|
||||
assert.Error(t, err, "input/output error")
|
||||
|
||||
run.rm(t, "testdoubleclose")
|
||||
}
|
||||
Loading…
Reference in New Issue