From 99695d57ab3ad81dc56f321fd6fdc0e8ef45b346 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Mon, 5 May 2014 19:52:52 +0100 Subject: [PATCH] Implement single file operations for all file systems --- README.md | 20 ++++++++++-- drive/drive.go | 81 ++++++++++++++++++++++++++++++++++------------ fs/fs.go | 12 +++++-- fs/limited.go | 88 ++++++++++++++++++++++++++++++++++++++++++++++++++ local/local.go | 10 ++++++ notes.txt | 6 ++-- rclone.go | 2 +- s3/s3.go | 27 ++++++++++++---- swift/swift.go | 26 +++++++++++---- 9 files changed, 228 insertions(+), 44 deletions(-) create mode 100644 fs/limited.go diff --git a/README.md b/README.md index 45280c795..5177b5122 100644 --- a/README.md +++ b/README.md @@ -143,7 +143,8 @@ Swift / Rackspace cloudfiles / Memset Memstore ---------------------------------------------- Paths are specified as remote:container (or remote: for the `lsd` -command.) +command.) You may put subdirectories in too, eg +`remote:container/path/to/dir`. So to copy a local directory to a swift container called backup: @@ -159,7 +160,8 @@ os.Stat) for an object. Amazon S3 --------- -Paths are specified as remote:bucket +Paths are specified as remote:bucket. You may put subdirectories in +too, eg `remote:bucket/path/to/dir`. So to copy a local directory to a s3 container called backup @@ -183,6 +185,19 @@ To copy a local directory to a drive directory called backup Google drive stores modification times accurate to 1 ms. +Single file copies +------------------ + +Rclone can copy single files + + rclone src:path/to/file dst:path/dir + +Or + + rclone src:path/to/file dst:path/to/file + +Note that you can't rename the file if you are copying from one file to another. + License ------- @@ -192,7 +207,6 @@ COPYING file included in this package). Bugs ---- - * Doesn't sync individual files yet, only directories. * Drive: Sometimes get: Failed to copy: Upload failed: googleapi: Error 403: Rate Limit Exceeded * quota is 100.0 requests/second/user * Empty directories left behind with Local and Drive diff --git a/drive/drive.go b/drive/drive.go index 6e88613de..5e319077e 100644 --- a/drive/drive.go +++ b/drive/drive.go @@ -135,14 +135,15 @@ func (name tokenCache) PutToken(token *oauth.Token) error { // FsDrive represents a remote drive server type FsDrive struct { - svc *drive.Service // the connection to the drive server - root string // the path we are working on - client *http.Client // authorized client - about *drive.About // information about the drive, including the root - rootId string // Id of the root directory - foundRoot sync.Once // Whether we need to find the root directory or not - dirCache dirCache // Map of directory path to directory id - findDirLock sync.Mutex // Protect findDir from concurrent use + svc *drive.Service // the connection to the drive server + root string // the path we are working on + client *http.Client // authorized client + about *drive.About // information about the drive, including the root + rootId string // Id of the root directory + foundRoot bool // Whether we have found the root or not + findRootLock sync.Mutex // Protect findRoot from concurrent use + dirCache dirCache // Map of directory path to directory id + findDirLock sync.Mutex // Protect findDir from concurrent use } // FsObjectDrive describes a drive object @@ -305,7 +306,10 @@ func NewFs(name, path string) (fs.Fs, error) { if err != nil { return nil, err } - f := &FsDrive{root: root, dirCache: newDirCache()} + f := &FsDrive{ + root: root, + dirCache: newDirCache(), + } // Try to pull the token from the cache; if this fails, we need to get one. token, err := t.Config.TokenCache.Token() @@ -331,14 +335,33 @@ func NewFs(name, path string) (fs.Fs, error) { f.rootId = f.about.RootFolderId // Put the root directory in f.dirCache.Put("", f.rootId) + // Find the current root + err = f.findRoot(false) + if err != nil { + // Assume it is a file + newRoot, remote := splitPath(root) + newF := *f + newF.root = newRoot + // Make new Fs which is the parent + err = newF.findRoot(false) + if err != nil { + // No root so return old f + return f, nil + } + obj, err := newF.newFsObjectWithInfo(remote, nil) + if err != nil { + // File doesn't exist so return old f + return f, nil + } + // return a Fs Limited to this object + return fs.NewLimited(&newF, obj), nil + } // fmt.Printf("Root id %s", f.rootId) return f, nil } // Return an FsObject from a path -// -// May return nil if an error occurred -func (f *FsDrive) NewFsObjectWithInfo(remote string, info *drive.File) fs.Object { +func (f *FsDrive) newFsObjectWithInfo(remote string, info *drive.File) (fs.Object, error) { fs := &FsObjectDrive{ drive: f, remote: remote, @@ -349,9 +372,18 @@ func (f *FsDrive) NewFsObjectWithInfo(remote string, info *drive.File) fs.Object err := fs.readMetaData() // reads info and meta, returning an error if err != nil { // logged already fs.Debug("Failed to read info: %s", err) - return nil + return nil, err } } + return fs, nil +} + +// Return an FsObject from a path +// +// May return nil if an error occurred +func (f *FsDrive) NewFsObjectWithInfo(remote string, info *drive.File) fs.Object { + fs, _ := f.newFsObjectWithInfo(remote, info) + // Errors have already been logged return fs } @@ -585,14 +617,21 @@ func (f *FsDrive) _findDir(path string, create bool) (pathId string, err error) // // If create is set it will make the directory if not found func (f *FsDrive) findRoot(create bool) error { - var err error - f.foundRoot.Do(func() { - f.rootId, err = f.findDir(f.root, create) - f.dirCache.Flush() - // Put the root directory in - f.dirCache.Put("", f.rootId) - }) - return err + f.findRootLock.Lock() + defer f.findRootLock.Unlock() + if f.foundRoot { + return nil + } + rootId, err := f.findDir(f.root, create) + if err != nil { + return err + } + f.rootId = rootId + f.dirCache.Flush() + // Put the root directory in + f.dirCache.Put("", f.rootId) + f.foundRoot = true + return nil } // Walk the path returning a channel of FsObjects diff --git a/fs/fs.go b/fs/fs.go index aa1e2df80..93434c689 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -18,9 +18,15 @@ var ( // Filesystem info type FsInfo struct { - Name string // name of this fs - NewFs func(string, string) (Fs, error) // create a new file system - Config func(string) // function to call to help with config + // Name of this fs + Name string + // Create a new file system. If root refers to an existing + // object, then it should return a Fs which only returns that + // object. + NewFs func(name string, root string) (Fs, error) + // Function to call to help with config + Config func(string) + // Options for the Fs configuration Options []Option } diff --git a/fs/limited.go b/fs/limited.go new file mode 100644 index 000000000..e38ef7e6d --- /dev/null +++ b/fs/limited.go @@ -0,0 +1,88 @@ +package fs + +import ( + "fmt" + "io" + "time" +) + +// This defines a Limited Fs which can only return the Objects passed in from the Fs passed in +type Limited struct { + objects []Object + fs Fs +} + +// NewLimited maks a limited Fs limited to the objects passed in +func NewLimited(fs Fs, objects ...Object) Fs { + f := &Limited{ + objects: objects, + fs: fs, + } + return f +} + +// String returns a description of the FS +func (f *Limited) String() string { + return fmt.Sprintf("%s limited to %d objects", f.fs.String(), len(f.objects)) +} + +// List the Fs into a channel +func (f *Limited) List() ObjectsChan { + out := make(ObjectsChan, Config.Checkers) + go func() { + for _, obj := range f.objects { + out <- obj + } + close(out) + }() + return out +} + +// List the Fs directories/buckets/containers into a channel +func (f *Limited) ListDir() DirChan { + out := make(DirChan, Config.Checkers) + close(out) + return out +} + +// Find the Object at remote. Returns nil if can't be found +func (f *Limited) NewFsObject(remote string) Object { + for _, obj := range f.objects { + if obj.Remote() == remote { + return obj + } + } + return nil +} + +// Put in to the remote path with the modTime given of the given size +// +// May create the object even if it returns an error - if so +// will return the object and the error, otherwise will return +// nil and the error +func (f *Limited) Put(in io.Reader, remote string, modTime time.Time, size int64) (Object, error) { + obj := f.NewFsObject(remote) + if obj == nil { + return nil, fmt.Errorf("Can't create %q in limited fs", remote) + } + return obj, obj.Update(in, modTime, size) +} + +// Make the directory (container, bucket) +func (f *Limited) Mkdir() error { + // All directories are already made - just ignore + return nil +} + +// Remove the directory (container, bucket) if empty +func (f *Limited) Rmdir() error { + return fmt.Errorf("Can't rmdir in limited fs") +} + +// Precision of the ModTimes in this Fs +func (f *Limited) Precision() time.Duration { + return f.fs.Precision() +} + +// Check the interfaces are satisfied +var _ Fs = &Limited{} diff --git a/local/local.go b/local/local.go index b6da7b36b..506f59501 100644 --- a/local/local.go +++ b/local/local.go @@ -45,6 +45,16 @@ type FsObjectLocal struct { func NewFs(name, root string) (fs.Fs, error) { root = path.Clean(root) f := &FsLocal{root: root} + // Check to see if this points to a file + fi, err := os.Lstat(f.root) + if err == nil && fi.Mode().IsRegular() { + // It is a file, so use the parent as the root + remote := path.Base(root) + f.root = path.Dir(root) + obj := f.NewFsObject(remote) + // return a Fs Limited to this object + return fs.NewLimited(f, obj), nil + } return f, nil } diff --git a/notes.txt b/notes.txt index 9c85bda5c..dab04d098 100644 --- a/notes.txt +++ b/notes.txt @@ -1,5 +1,4 @@ Todo - * FIXME: ls without an argument for buckets/containers? * FIXME: More -dry-run checks for object transfer * Might be quicker to check md5sums first? for swift <-> swift certainly, and maybe for small files * swift: Ignoring the pseudo directories @@ -12,7 +11,6 @@ Todo * make Account do progress meter * Make logging controllable with flags (mostly done) * -timeout: Make all timeouts be settable with command line parameters - * Check the locking in swift module! * Windows paths? Do we need to translate / and \? * Make a fs.Errorf and count errors and log them at a different level * Add max object size to fs metadata - 5GB for swift, infinite for local, ? for s3 @@ -22,7 +20,6 @@ Ideas * could do encryption - put IV into metadata? * optimise remote copy container to another container using remote copy if local is same as remote - use an optional Copier interface - * Allow subpaths container:/sub/path * support * sftp * scp @@ -35,6 +32,8 @@ Need to make directory objects otherwise can't upload an empty directory * Or could upload empty directories only? * Can't purge a local filesystem because it leaves the directories behind +Copying a single file? Or maybe with a glob pattern? Could do with LimitedFs + s3 * Can maybe set last modified? * https://forums.aws.amazon.com/message.jspa?messageID=214062 @@ -43,6 +42,7 @@ s3 Bugs * Non verbose - not sure number transferred got counted up? CHECK + * When doing copy it recurses the whole of the destination FS which isn't necessary Making a release * go build ./... diff --git a/rclone.go b/rclone.go index 2357a3193..7ef6fbeb5 100644 --- a/rclone.go +++ b/rclone.go @@ -106,7 +106,7 @@ var Commands = []Command{ Name: "lsd", ArgsHelp: "[remote://path]", Help: ` - List all directoryes/objects/buckets in the the path.`, + List all directories/containers/buckets in the the path.`, Run: func(fdst, fsrc fs.Fs) { err := fs.ListDir(fdst) if err != nil { diff --git a/s3/s3.go b/s3/s3.go index 468ae932b..b38994767 100644 --- a/s3/s3.go +++ b/s3/s3.go @@ -188,8 +188,8 @@ func s3Connection(name string) (*s3.S3, error) { } // NewFsS3 contstructs an FsS3 from the path, bucket:path -func NewFs(name, path string) (fs.Fs, error) { - bucket, directory, err := s3ParsePath(path) +func NewFs(name, root string) (fs.Fs, error) { + bucket, directory, err := s3ParsePath(root) if err != nil { return nil, err } @@ -197,10 +197,6 @@ func NewFs(name, path string) (fs.Fs, error) { if err != nil { return nil, err } - // FIXME - check if it is a file before doing this and make a limited fs - if directory != "" { - directory += "/" - } f := &FsS3{ c: c, bucket: bucket, @@ -208,6 +204,23 @@ func NewFs(name, path string) (fs.Fs, error) { perm: s3.Private, // FIXME need user to specify root: directory, } + if f.root != "" { + f.root += "/" + // Check to see if the object exists + _, err = f.b.Head(directory, nil) + if err == nil { + remote := path.Base(directory) + f.root = path.Dir(directory) + if f.root == "." { + f.root = "" + } else { + f.root += "/" + } + obj := f.NewFsObject(remote) + // return a Fs Limited to this object + return fs.NewLimited(f, obj), nil + } + } return f, nil } @@ -328,7 +341,7 @@ func (f *FsS3) ListDir() fs.DirChan { } }() } else { - // List the directories in the path in the container + // List the directories in the path in the bucket go func() { defer close(out) f.list(true, func(remote string, object *s3.Key) { diff --git a/swift/swift.go b/swift/swift.go index 2e6bc149b..1441af1f8 100644 --- a/swift/swift.go +++ b/swift/swift.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "path" "regexp" "strings" "time" @@ -118,8 +119,8 @@ func swiftConnection(name string) (*swift.Connection, error) { } // NewFs contstructs an FsSwift from the path, container:path -func NewFs(name, path string) (fs.Fs, error) { - container, directory, err := parsePath(path) +func NewFs(name, root string) (fs.Fs, error) { + container, directory, err := parsePath(root) if err != nil { return nil, err } @@ -127,15 +128,28 @@ func NewFs(name, path string) (fs.Fs, error) { if err != nil { return nil, err } - // FIXME - check if it is a file before doing this and make a limited fs - if directory != "" { - directory += "/" - } f := &FsSwift{ c: *c, container: container, root: directory, } + if f.root != "" { + f.root += "/" + // Check to see if the object exists + _, _, err = f.c.Object(container, directory) + if err == nil { + remote := path.Base(directory) + f.root = path.Dir(directory) + if f.root == "." { + f.root = "" + } else { + f.root += "/" + } + obj := f.NewFsObject(remote) + // return a Fs Limited to this object + return fs.NewLimited(f, obj), nil + } + } return f, nil }