diff --git a/README.md b/README.md index abdb41a50..e3ac23845 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ Rclone is a command line program to sync files and directories to and from * Openstack Swift / Rackspace cloud files / Memset Memstore * Dropbox * Google Cloud Storage + * Amazon Cloud Drive * The local filesystem Features diff --git a/amazonclouddrive/amazonclouddrive.go b/amazonclouddrive/amazonclouddrive.go new file mode 100644 index 000000000..257283a8c --- /dev/null +++ b/amazonclouddrive/amazonclouddrive.go @@ -0,0 +1,622 @@ +// Amazon Cloud Drive interface +package amazonclouddrive + +/* + +FIXME make searching for directory in id and file in id more efficient +- use the name: search parameter - remember the escaping rules +- use Folder GetNode and GetFile + +FIXME make the default for no files and no dirs be (FILE & FOLDER) so +we ignore assets completely! + +FIXME detect 429 errors and return error with fs.RetryErrorf? + +*/ + +import ( + "fmt" + "io" + "log" + "regexp" + "strings" + "time" + + "github.com/ncw/go-acd" + "github.com/ncw/rclone/dircache" + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/oauthutil" + "golang.org/x/oauth2" +) + +const ( + rcloneClientID = "amzn1.application-oa2-client.6bf18d2d1f5b485c94c8988bb03ad0e7" + rcloneClientSecret = "k8/NyszKm5vEkZXAwsbGkd6C3NrbjIqMg4qEhIeF14Szub2wur+/teS3ubXgsLe9//+tr/qoqK+lq6mg8vWkoA==" + bindAddress = "127.0.0.1:53682" + redirectURL = "http://" + bindAddress + "/" + folderKind = "FOLDER" + fileKind = "FILE" + assetKind = "ASSET" + statusAvailable = "AVAILABLE" + timeFormat = time.RFC3339 // 2014-03-07T22:31:12.173Z +) + +// Globals +var ( + // Description of how to auth for this app + acdConfig = &oauth2.Config{ + Scopes: []string{"clouddrive:read_all", "clouddrive:write"}, + Endpoint: oauth2.Endpoint{ + AuthURL: "https://www.amazon.com/ap/oa", + TokenURL: "https://api.amazon.com/auth/o2/token", + }, + ClientID: rcloneClientID, + ClientSecret: fs.Reveal(rcloneClientSecret), + RedirectURL: redirectURL, + } + FIXME = fmt.Errorf("FIXME not implemented") +) + +// Register with Fs +func init() { + fs.Register(&fs.FsInfo{ + Name: "amazon cloud drive", + NewFs: NewFs, + Config: func(name string) { + err := oauthutil.ConfigWithWebserver(name, acdConfig, bindAddress) + if err != nil { + log.Fatalf("Failed to configure token: %v", err) + } + }, + Options: []fs.Option{{ + Name: "client_id", + Help: "Amazon Application Client Id - leave blank to use rclone's.", + }, { + Name: "client_secret", + Help: "Amazon Application Client Secret - leave blank to use rclone's.", + }}, + }) +} + +// FsAcd represents a remote acd server +type FsAcd struct { + name string // name of this remote + c *acd.Client // the connection to the acd server + root string // the path we are working on + dirCache *dircache.DirCache // Map of directory path to directory id +} + +// FsObjectAcd describes a acd object +// +// Will definitely have info but maybe not meta +type FsObjectAcd struct { + acd *FsAcd // what this object is part of + remote string // The remote path + info *acd.Node // Info from the acd object if known +} + +// ------------------------------------------------------------ + +// The name of the remote (as passed into NewFs) +func (f *FsAcd) Name() string { + return f.name +} + +// The root of the remote (as passed into NewFs) +func (f *FsAcd) Root() string { + return f.root +} + +// String converts this FsAcd to a string +func (f *FsAcd) String() string { + return fmt.Sprintf("Amazon cloud drive root '%s'", f.root) +} + +// Pattern to match a acd path +var matcher = regexp.MustCompile(`^([^/]*)(.*)$`) + +// parsePath parses an acd 'url' +func parsePath(path string) (root string) { + root = strings.Trim(path, "/") + return +} + +// NewFs contstructs an FsAcd from the path, container:path +func NewFs(name, root string) (fs.Fs, error) { + root = parsePath(root) + oAuthClient, err := oauthutil.NewClient(name, acdConfig) + if err != nil { + log.Fatalf("Failed to configure amazon cloud drive: %v", err) + } + + c := acd.NewClient(oAuthClient) + c.UserAgent = fs.UserAgent + f := &FsAcd{ + name: name, + root: root, + c: c, + } + + // Update endpoints + _, _, err = f.c.Account.GetEndpoints() + if err != nil { + return nil, fmt.Errorf("Failed to get endpoints: %v", err) + } + + // Get rootID + rootInfo, _, err := f.c.Nodes.GetRoot() + if err != nil || rootInfo.Id == nil { + return nil, fmt.Errorf("Failed to get root: %v", err) + } + + f.dirCache = dircache.New(root, *rootInfo.Id, f) + + // Find the current root + err = f.dirCache.FindRoot(false) + if err != nil { + // Assume it is a file + newRoot, remote := dircache.SplitPath(root) + newF := *f + newF.dirCache = dircache.New(newRoot, *rootInfo.Id, &newF) + newF.root = newRoot + // Make new Fs which is the parent + err = newF.dirCache.FindRoot(false) + if err != nil { + // No root so return old f + return f, nil + } + obj := newF.newFsObjectWithInfo(remote, nil) + if obj == nil { + // File doesn't exist so return old f + return f, nil + } + // return a Fs Limited to this object + return fs.NewLimited(&newF, obj), nil + } + return f, nil +} + +// Return an FsObject from a path +// +// May return nil if an error occurred +func (f *FsAcd) newFsObjectWithInfo(remote string, info *acd.Node) fs.Object { + o := &FsObjectAcd{ + acd: f, + remote: remote, + } + if info != nil { + // Set info but not meta + o.info = info + } else { + err := o.readMetaData() // reads info and meta, returning an error + if err != nil { + // logged already FsDebug("Failed to read info: %s", err) + return nil + } + } + return o +} + +// Return an FsObject from a path +// +// May return nil if an error occurred +func (f *FsAcd) NewFsObject(remote string) fs.Object { + return f.newFsObjectWithInfo(remote, nil) +} + +// FindLeaf finds a directory of name leaf in the folder with ID pathId +func (f *FsAcd) FindLeaf(pathId, leaf string) (pathIdOut string, found bool, err error) { + //fs.Debug(f, "FindLeaf(%q, %q)", pathId, leaf) + folder := acd.FolderFromId(pathId, f.c.Nodes) + subFolder, _, err := folder.GetFolder(leaf) + if err != nil { + if err == acd.ErrorNodeNotFound { + //fs.Debug(f, "...Not found") + return "", false, nil + } + //fs.Debug(f, "...Error %v", err) + return "", false, err + } + if subFolder.Status != nil && *subFolder.Status != statusAvailable { + fs.Debug(f, "Ignoring folder %q in state %q", *subFolder.Status) + time.Sleep(1 * time.Second) // FIXME wait for problem to go away! + return "", false, nil + } + //fs.Debug(f, "...Found(%q, %v)", *subFolder.Id, leaf) + return *subFolder.Id, true, nil +} + +// CreateDir makes a directory with pathId as parent and name leaf +func (f *FsAcd) CreateDir(pathId, leaf string) (newId string, err error) { + //fs.Debug(f, "CreateDir(%q, %q)", pathId, leaf) + folder := acd.FolderFromId(pathId, f.c.Nodes) + info, _, err := folder.CreateFolder(leaf) + if err != nil { + fs.Debug(f, "...Error %v", err) + return "", err + } + //fs.Debug(f, "...Id %q", *info.Id) + return *info.Id, nil +} + +// list the objects into the function supplied +// +// If directories is set it only sends directories +// User function to process a File item from listAll +// +// Should return true to finish processing +type listAllFn func(*acd.Node) bool + +// Lists the directory required calling the user function on each item found +// +// If the user fn ever returns true then it early exits with found = true +func (f *FsAcd) listAll(dirId string, title string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) { + query := "parents:" + dirId + if directoriesOnly { + query += " AND kind:" + folderKind + } else if filesOnly { + query += " AND kind:" + fileKind + } else { + // FIXME none of these work + //query += " AND kind:(" + fileKind + " OR " + folderKind + ")" + //query += " AND (kind:" + fileKind + " OR kind:" + folderKind + ")" + } + opts := acd.NodeListOptions{ + Filters: query, + } + var nodes []*acd.Node +OUTER: + for { + nodes, _, err = f.c.Nodes.GetNodes(&opts) + if err != nil { + fs.Stats.Error() + fs.ErrorLog(f, "Couldn't list files: %v", err) + break + } + if nodes == nil { + break + } + for _, node := range nodes { + if node.Name != nil && node.Id != nil && node.Kind != nil && node.Status != nil { + // Ignore nodes if not AVAILABLE + if *node.Status != statusAvailable { + continue + } + if fn(node) { + found = true + break OUTER + } + } + } + } + return +} + +// Path should be directory path either "" or "path/" +// +// List the directory using a recursive list from the root +// +// This fetches the minimum amount of stuff but does more API calls +// which makes it slow +func (f *FsAcd) listDirRecursive(dirId string, path string, out fs.ObjectsChan) error { + var subError error + // Make the API request + _, err := f.listAll(dirId, "", false, false, func(node *acd.Node) bool { + // Recurse on directories + // FIXME should do this in parallel + // use a wg to sync then collect error + switch *node.Kind { + case folderKind: + subError = f.listDirRecursive(*node.Id, path+*node.Name+"/", out) + if subError != nil { + return true + } + case fileKind: + if fs := f.newFsObjectWithInfo(path+*node.Name, node); fs != nil { + out <- fs + } + default: + // ignore ASSET etc + } + return false + }) + if err != nil { + return err + } + if subError != nil { + return subError + } + return nil +} + +// Walk the path returning a channel of FsObjects +func (f *FsAcd) List() fs.ObjectsChan { + out := make(fs.ObjectsChan, fs.Config.Checkers) + go func() { + defer close(out) + err := f.dirCache.FindRoot(false) + if err != nil { + fs.Stats.Error() + fs.ErrorLog(f, "Couldn't find root: %s", err) + } else { + err = f.listDirRecursive(f.dirCache.RootID(), "", out) + if err != nil { + fs.Stats.Error() + fs.ErrorLog(f, "List failed: %s", err) + } + } + }() + return out +} + +// Lists the containers +func (f *FsAcd) ListDir() fs.DirChan { + out := make(fs.DirChan, fs.Config.Checkers) + go func() { + defer close(out) + err := f.dirCache.FindRoot(false) + if err != nil { + fs.Stats.Error() + fs.ErrorLog(f, "Couldn't find root: %s", err) + } else { + _, err := f.listAll(f.dirCache.RootID(), "", true, false, func(item *acd.Node) bool { + dir := &fs.Dir{ + Name: *item.Name, + Bytes: -1, + Count: -1, + } + dir.When, _ = time.Parse(timeFormat, *item.ModifiedDate) + out <- dir + return false + }) + if err != nil { + fs.Stats.Error() + fs.ErrorLog(f, "ListDir failed: %s", err) + } + } + }() + return out +} + +// Put the object into the container +// +// Copy the reader in to the new object which is returned +// +// The new object may have been created if an error is returned +func (f *FsAcd) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) { + // Temporary FsObject under construction + o := &FsObjectAcd{ + acd: f, + remote: remote, + } + leaf, directoryID, err := f.dirCache.FindPath(remote, true) + if err != nil { + return nil, err + } + folder := acd.FolderFromId(directoryID, o.acd.c.Nodes) + info, _, err := folder.Put(in, leaf) + if err != nil { + return nil, err + } + o.info = info.Node + return o, nil +} + +// Mkdir creates the container if it doesn't exist +func (f *FsAcd) Mkdir() error { + return f.dirCache.FindRoot(true) +} + +// purgeCheck remotes the root directory, if check is set then it +// refuses to do so if it has anything in +func (f *FsAcd) purgeCheck(check bool) error { + if f.root == "" { + return fmt.Errorf("Can't purge root directory") + } + dc := f.dirCache + err := dc.FindRoot(false) + if err != nil { + return err + } + rootID := dc.RootID() + + if check { + // check directory is empty + empty := true + _, err := f.listAll(rootID, "", false, false, func(node *acd.Node) bool { + switch *node.Kind { + case folderKind: + empty = false + return true + case fileKind: + empty = false + return true + default: + fs.Debug("Found ASSET %s", *node.Id) + } + return false + }) + if err != nil { + return err + } + if !empty { + return fmt.Errorf("Directory not empty") + } + } + + node := acd.NodeFromId(rootID, f.c.Nodes) + _, err = node.Trash() + if err != nil { + return err + } + + f.dirCache.ResetRoot() + if err != nil { + return err + } + return nil +} + +// Rmdir deletes the root folder +// +// Returns an error if it isn't empty +func (f *FsAcd) Rmdir() error { + return f.purgeCheck(true) +} + +// Return the precision +func (f *FsAcd) Precision() time.Duration { + return fs.ModTimeNotSupported +} + +// Copy src to this remote using server side copy operations. +// +// This is stored with the remote path given +// +// It returns the destination Object and a possible error +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantCopy +//func (f *FsAcd) Copy(src fs.Object, remote string) (fs.Object, error) { +// srcObj, ok := src.(*FsObjectAcd) +// if !ok { +// fs.Debug(src, "Can't copy - not same remote type") +// return nil, fs.ErrorCantCopy +// } +// srcFs := srcObj.acd +// _, err := f.c.ObjectCopy(srcFs.container, srcFs.root+srcObj.remote, f.container, f.root+remote, nil) +// if err != nil { +// return nil, err +// } +// return f.NewFsObject(remote), nil +//} + +// Purge deletes all the files and the container +// +// Optional interface: Only implement this if you have a way of +// deleting all the files quicker than just running Remove() on the +// result of List() +func (f *FsAcd) Purge() error { + return f.purgeCheck(false) +} + +// ------------------------------------------------------------ + +// Return the parent Fs +func (o *FsObjectAcd) Fs() fs.Fs { + return o.acd +} + +// Return a string version +func (o *FsObjectAcd) String() string { + if o == nil { + return "" + } + return o.remote +} + +// Return the remote path +func (o *FsObjectAcd) Remote() string { + return o.remote +} + +// Md5sum returns the Md5sum of an object returning a lowercase hex string +func (o *FsObjectAcd) Md5sum() (string, error) { + if o.info.ContentProperties.Md5 != nil { + return *o.info.ContentProperties.Md5, nil + } + return "", nil +} + +// Size returns the size of an object in bytes +func (o *FsObjectAcd) Size() int64 { + return int64(*o.info.ContentProperties.Size) +} + +// readMetaData gets the metadata if it hasn't already been fetched +// +// it also sets the info +func (o *FsObjectAcd) readMetaData() (err error) { + if o.info != nil { + return nil + } + leaf, directoryID, err := o.acd.dirCache.FindPath(o.remote, false) + if err != nil { + return err + } + folder := acd.FolderFromId(directoryID, o.acd.c.Nodes) + info, _, err := folder.GetFile(leaf) + if err != nil { + fs.Debug(o, "Failed to read info: %s", err) + return err + } + o.info = info.Node + return nil +} + +// ModTime returns the modification time of the object +// +// +// It attempts to read the objects mtime and if that isn't present the +// LastModified returned in the http headers +func (o *FsObjectAcd) ModTime() time.Time { + err := o.readMetaData() + if err != nil { + fs.Log(o, "Failed to read metadata: %s", err) + return time.Now() + } + modTime, err := time.Parse(timeFormat, *o.info.ModifiedDate) + if err != nil { + fs.Log(o, "Failed to read mtime from object: %s", err) + return time.Now() + } + return modTime +} + +// Sets the modification time of the local fs object +func (o *FsObjectAcd) SetModTime(modTime time.Time) { + // FIXME not implemented + return +} + +// Is this object storable +func (o *FsObjectAcd) Storable() bool { + return true +} + +// Open an object for read +func (o *FsObjectAcd) Open() (in io.ReadCloser, err error) { + file := acd.File{Node: o.info} + in, _, err = file.Open() + return in, err +} + +// Update the object with the contents of the io.Reader, modTime and size +// +// The new object may have been created if an error is returned +func (o *FsObjectAcd) Update(in io.Reader, modTime time.Time, size int64) error { + file := acd.File{Node: o.info} + info, _, err := file.Overwrite(in) + if err != nil { + return err + } + o.info = info.Node + return nil +} + +// Remove an object +func (o *FsObjectAcd) Remove() error { + _, err := o.info.Trash() + return err +} + +// Check the interfaces are satisfied +var ( + _ fs.Fs = (*FsAcd)(nil) + _ fs.Purger = (*FsAcd)(nil) + // _ fs.Copier = (*FsAcd)(nil) + // _ fs.Mover = (*FsAcd)(nil) + // _ fs.DirMover = (*FsAcd)(nil) + _ fs.Object = (*FsObjectAcd)(nil) +) diff --git a/amazonclouddrive/amazonclouddrive_test.go b/amazonclouddrive/amazonclouddrive_test.go new file mode 100644 index 000000000..b3c45a051 --- /dev/null +++ b/amazonclouddrive/amazonclouddrive_test.go @@ -0,0 +1,56 @@ +// Test AmazonCloudDrive filesystem interface +// +// Automatically generated - DO NOT EDIT +// Regenerate with: make gen_tests +package amazonclouddrive_test + +import ( + "testing" + + "github.com/ncw/rclone/amazonclouddrive" + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fstest/fstests" +) + +func init() { + fstests.NilObject = fs.Object((*amazonclouddrive.FsObjectAcd)(nil)) + fstests.RemoteName = "TestAmazonCloudDrive:" +} + +// Generic tests for the Fs +func TestInit(t *testing.T) { fstests.TestInit(t) } +func TestFsString(t *testing.T) { fstests.TestFsString(t) } +func TestFsRmdirEmpty(t *testing.T) { fstests.TestFsRmdirEmpty(t) } +func TestFsRmdirNotFound(t *testing.T) { fstests.TestFsRmdirNotFound(t) } +func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) } +func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) } +func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) } +func TestFsNewFsObjectNotFound(t *testing.T) { fstests.TestFsNewFsObjectNotFound(t) } +func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) } +func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) } +func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) } +func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) } +func TestFsListRoot(t *testing.T) { fstests.TestFsListRoot(t) } +func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } +func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) } +func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } +func TestFsCopy(t *testing.T) { fstests.TestFsCopy(t) } +func TestFsMove(t *testing.T) { fstests.TestFsMove(t) } +func TestFsDirMove(t *testing.T) { fstests.TestFsDirMove(t) } +func TestFsRmdirFull(t *testing.T) { fstests.TestFsRmdirFull(t) } +func TestFsPrecision(t *testing.T) { fstests.TestFsPrecision(t) } +func TestObjectString(t *testing.T) { fstests.TestObjectString(t) } +func TestObjectFs(t *testing.T) { fstests.TestObjectFs(t) } +func TestObjectRemote(t *testing.T) { fstests.TestObjectRemote(t) } +func TestObjectMd5sum(t *testing.T) { fstests.TestObjectMd5sum(t) } +func TestObjectModTime(t *testing.T) { fstests.TestObjectModTime(t) } +func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } +func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } +func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } +func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) } +func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) } +func TestLimitedFs(t *testing.T) { fstests.TestLimitedFs(t) } +func TestLimitedFsNotFound(t *testing.T) { fstests.TestLimitedFsNotFound(t) } +func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) } +func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) } +func TestFinalise(t *testing.T) { fstests.TestFinalise(t) } diff --git a/docs/content/about.md b/docs/content/about.md index 20afbf2df..b0c8b35ad 100644 --- a/docs/content/about.md +++ b/docs/content/about.md @@ -1,8 +1,8 @@ --- title: "Rclone" -description: "rclone syncs files to and from Google Drive, S3, Swift, Cloudfiles, Dropbox and Google Cloud Storage." +description: "rclone syncs files to and from Google Drive, S3, Swift, Cloudfiles, Dropbox, Google Cloud Storage and Amazon Cloud Drive." type: page -date: "2014-07-17" +date: "2015-09-06" groups: ["about"] --- @@ -18,6 +18,7 @@ Rclone is a command line program to sync files and directories to and from * Openstack Swift / Rackspace cloud files / Memset Memstore * Dropbox * Google Cloud Storage + * Amazon Cloud Drive * The local filesystem Features diff --git a/docs/content/amazonclouddrive.md b/docs/content/amazonclouddrive.md new file mode 100644 index 000000000..492806ea0 --- /dev/null +++ b/docs/content/amazonclouddrive.md @@ -0,0 +1,104 @@ +--- +title: "Amazon Cloud Drive" +description: "Rclone docs for Amazon Cloud Drive" +date: "2015-09-06" +--- + + Amazon Cloud Drive +----------------------------------------- + +Paths are specified as `remote:path` + +Paths may be as deep as required, eg `remote:directory/subdirectory`. + +The initial setup for Amazon cloud drive involves getting a token from +Amazon which you need to do in your browser. `rclone config` walks +you through it. + +Here is an example of how to make a remote called `remote`. First run: + + rclone config + +This will guide you through an interactive setup process: + +``` +n) New remote +d) Delete remote +q) Quit config +e/n/d/q> n +name> remote +What type of source is it? +Choose a number from below + 1) amazon cloud drive + 2) drive + 3) dropbox + 4) google cloud storage + 5) local + 6) s3 + 7) swift +type> 1 +Amazon Application Client Id - leave blank to use rclone's. +client_id> +Amazon Application Client Secret - leave blank to use rclone's. +client_secret> +Remote config +If your browser doesn't open automatically go to the following link +https://www.amazon.com/ap/oa?client_id=amzn1.application-oa2-client.xxxxxxxxxxxxxxx&redirect_uri=http%3A%2F%2F127.0.0.1%3A53682%2F&response_type=code&scope=clouddrive%3Aread_all+clouddrive%3Awrite&state=xxxxxxxxxxxxxxxxx +Log in, then cut and paste the token that is returned from the browser here +Enter verification code> xxxxxxxxxxxxxxxxxxxx +-------------------- +[remote] +client_id = +client_secret = +token = {"access_token":"xxxxxxxxxxxxxxxxxxxxxxx","token_type":"bearer","refresh_token":"xxxxxxxxxxxxxxxxxx","expiry":"2015-09-06T16:07:39.658438471+01:00"} +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +Note that rclone runs a webserver on your local machine to collect the +token as returned from Amazon. This is only run from the moment it +opens your browser to the moment you cut and paste the verification +code. This is on `http://127.0.0.1:53682/` and this it may require +you to unblock it temporarily if you are running a host firewall. + +Once configured you can then use `rclone` like this, + +List directories in top level of your Amazon cloud drive + + rclone lsd remote: + +List all the files in your Amazon cloud drive + + rclone ls remote: + +To copy a local directory to an Amazon cloud drive directory called backup + + rclone copy /home/source remote:backup + +### Modified time and MD5SUMs ### + +Amazon cloud drive doesn't allow modification times to be changed via +the API so these won't be accurate or used for syncing. + +It does store MD5SUMs so for a more accurate sync, you can use the +`--checksum` flag. + +### Deleting files ### + +Any files you delete with rclone will end up in the trash. Amazon +don't provide an API to permanently delete files, nor to empty the +trash, so you will have to do that with one of Amazon's apps or via +the Amazon cloud drive website. + +### Limitations ### + +Note that Amazon cloud drive is case sensitive so you can't have a +file called "Hello.doc" and one called "hello.doc". + +Amazon cloud drive has rate limiting so you may notice errors in the +sync (429 errors). rclone will automatically retry the sync up to 3 +times by default (see `--retries` flag) which should hopefully work +around this problem. diff --git a/docs/content/overview.md b/docs/content/overview.md new file mode 100644 index 000000000..4b0761e5d --- /dev/null +++ b/docs/content/overview.md @@ -0,0 +1,71 @@ +--- +title: "Overview of cloud storage systems" +description: "Overview of cloud storage systems" +type: page +date: "2015-09-06" +--- + +# Overview of cloud storage systems # + +Each cloud storage system is slighly different. Rclone attempts to +provide a unified interface to them, but some underlying differences +show through. + +## Features ## + +Here is an overview of the major features of each cloud storage system. + +| Name | MD5SUM | ModTime | Case Sensitive | Duplicate Files | +| ---------------------- |:-------:|:-------:|:--------------:|:---------------:| +| Google Drive | Yes | Yes | No | Yes | +| Amazon S3 | Yes | Yes | No | No | +| Openstack Swift | Yes | Yes | No | No | +| Dropbox | No | No | Yes | No | +| Google Cloud Storage | Yes | Yes | No | No | +| Amazon Cloud Drive | Yes | No | Yes | No | +| The local filesystem | Yes | Yes | Depends | No | + +### MD5SUM ### + +The cloud storage system supports MD5SUMs of the objects. This +is used if available when transferring data as an integrity check and +can be specifically used with the `--checksum` flag in syncs and in +the `check` command. + +### ModTime ### + +The cloud storage system supports setting modification times on +objects. If it does then this enables a using the modification times +as part of the sync. If not then only the size will be checked by +default, though the MD5SUM can be checked with the `--checksum` flag. + +All cloud storage systems support some kind of date on the object and +these will be set when transferring from the cloud storage system. + +### Case Sensitive ### + +If a cloud storage systems is case sensitive then it is possible to +have two files which differ only in case, eg `file.txt` and +`FILE.txt`. If a cloud storage system is case insensitive then that +isn't possible. + +This can cause problems when syncing between a case insensitive +system and a case sensitive system. The symptom of this is that no +matter how many times you run the sync it never completes fully. + +The local filesystem may or may not be case sensitive depending on OS. + + * Windows - usuall case insensitive + * OSX - usually case insensitive, though it is possible to format case sensitive + * Linux - usually case sensitive, but there are case insensitive file systems (eg FAT formatted USB keys) + +Most of the time this doesn't cause any problems as people tend to +avoid files whose name differs only by case even on case sensitive +systems. + +### Duplicate files ### + +If a cloud storage system allows duplicate files then it can have two +objects with the same name. + +This confuses rclone greatly when syncing. diff --git a/docs/layouts/chrome/navbar.html b/docs/layouts/chrome/navbar.html index de7d870ff..814cb4286 100644 --- a/docs/layouts/chrome/navbar.html +++ b/docs/layouts/chrome/navbar.html @@ -28,11 +28,13 @@
  • Storage Systems
    • diff --git a/docs/static/css/custom.css b/docs/static/css/custom.css index a9bb3c03b..76101285b 100644 --- a/docs/static/css/custom.css +++ b/docs/static/css/custom.css @@ -4,4 +4,23 @@ body { footer { margin: 50px 0; -} \ No newline at end of file +} + +table { + background-color:#e0e0ff +} + +tbody td, th { + border: 1px solid black; + padding: 3px 7px 2px 7px; +} + +thead td, th { + border: 1px solid black; + padding: 3px 7px 2px 7px; + font-weight: bold; +} + +tbody tr:nth-child(odd) { + background-color:#d0d0ff +} diff --git a/fs/operations_test.go b/fs/operations_test.go index da9664b9b..2a9137cda 100644 --- a/fs/operations_test.go +++ b/fs/operations_test.go @@ -20,6 +20,7 @@ import ( "github.com/ncw/rclone/fstest" // Active file systems + _ "github.com/ncw/rclone/amazonclouddrive" _ "github.com/ncw/rclone/drive" _ "github.com/ncw/rclone/dropbox" _ "github.com/ncw/rclone/googlecloudstorage" diff --git a/fs/test_all.sh b/fs/test_all.sh index d20e0cbdb..cdf29c229 100755 --- a/fs/test_all.sh +++ b/fs/test_all.sh @@ -8,6 +8,11 @@ TestS3: TestDrive: TestGoogleCloudStorage: TestDropbox: +TestAmazonCloudDrive: +" + +REMOTES=" +TestAmazonCloudDrive: " function test_remote { diff --git a/fstest/fstests/gen_tests.go b/fstest/fstests/gen_tests.go index 2534e0e8a..ff5a7a24f 100644 --- a/fstest/fstests/gen_tests.go +++ b/fstest/fstests/gen_tests.go @@ -75,17 +75,10 @@ func init() { ` // Generate test file piping it through gofmt -func generateTestProgram(t *template.Template, fns []string, Fsname string) { +func generateTestProgram(t *template.Template, fns []string, Fsname, ObjectName string) { fsname := strings.ToLower(Fsname) TestName := "Test" + Fsname + ":" outfile := "../../" + fsname + "/" + fsname + "_test.go" - // Find last capitalised group to be object name - matcher := regexp.MustCompile(`([A-Z][a-z0-9]+)$`) - matches := matcher.FindStringSubmatch(Fsname) - if len(matches) == 0 { - log.Fatalf("Couldn't find object name in %q", Fsname) - } - ObjectName := matches[1] if fsname == "local" { TestName = "" @@ -133,11 +126,12 @@ func generateTestProgram(t *template.Template, fns []string, Fsname string) { func main() { fns := findTestFunctions() t := template.Must(template.New("main").Parse(testProgram)) - generateTestProgram(t, fns, "Local") - generateTestProgram(t, fns, "Swift") - generateTestProgram(t, fns, "S3") - generateTestProgram(t, fns, "Drive") - generateTestProgram(t, fns, "GoogleCloudStorage") - generateTestProgram(t, fns, "Dropbox") + generateTestProgram(t, fns, "Local", "Local") + generateTestProgram(t, fns, "Swift", "Swift") + generateTestProgram(t, fns, "S3", "S3") + generateTestProgram(t, fns, "Drive", "Drive") + generateTestProgram(t, fns, "GoogleCloudStorage", "Storage") + generateTestProgram(t, fns, "Dropbox", "Dropbox") + generateTestProgram(t, fns, "AmazonCloudDrive", "Acd") log.Printf("Done") } diff --git a/make_manual.py b/make_manual.py index 59987342a..836112f02 100755 --- a/make_manual.py +++ b/make_manual.py @@ -16,11 +16,13 @@ docs = [ "about.md", "install.md", "docs.md", + "overview.md", "drive.md", "s3.md", "swift.md", "dropbox.md", "googlecloudstorage.md", + "amazonclouddrive.md", "local.md", "changelog.md", "bugs.md", diff --git a/oauthutil/oauthutil.go b/oauthutil/oauthutil.go index 24b8b04f9..9a4496290 100644 --- a/oauthutil/oauthutil.go +++ b/oauthutil/oauthutil.go @@ -1,12 +1,16 @@ package oauthutil import ( + "crypto/rand" "encoding/json" "fmt" + "log" + "net" "net/http" "time" "github.com/ncw/rclone/fs" + "github.com/skratchdot/open-golang/open" "golang.org/x/net/context" "golang.org/x/oauth2" ) @@ -139,7 +143,9 @@ func NewClient(name string, config *oauth2.Config) (*http.Client, error) { } // Config does the initial creation of the token -func Config(name string, config *oauth2.Config) error { +// +// It runs an internal webserver to receive the results +func ConfigWithWebserver(name string, config *oauth2.Config, bindAddress string) error { // See if already have a token tokenString := fs.ConfigFile.MustValue(name, "token") if tokenString != "" { @@ -149,11 +155,30 @@ func Config(name string, config *oauth2.Config) error { } } + // Make random state + stateBytes := make([]byte, 16) + _, err := rand.Read(stateBytes) + if err != nil { + return err + } + state := fmt.Sprintf("%x", stateBytes) + + // Prepare webserver + server := authServer{ + state: state, + bindAddress: bindAddress, + } + if bindAddress != "" { + go server.Start() + defer server.Stop() + } + // Generate a URL for the user to visit for authorization. - authUrl := config.AuthCodeURL("state") - fmt.Printf("Go to the following link in your browser\n") + authUrl := config.AuthCodeURL(state) + _ = open.Start(authUrl) + fmt.Printf("If your browser doesn't open automatically go to the following link\n") fmt.Printf("%s\n", authUrl) - fmt.Printf("Log in, then type paste the token that is returned in the browser here\n") + fmt.Printf("Log in, then cut and paste the token that is returned from the browser here\n") // Read the code, and exchange it for a token. fmt.Printf("Enter verification code> ") @@ -164,3 +189,60 @@ func Config(name string, config *oauth2.Config) error { } return putToken(name, token) } + +// Config does the initial creation of the token +func Config(name string, config *oauth2.Config) error { + return ConfigWithWebserver(name, config, "") +} + +// Local web server for collecting auth +type authServer struct { + state string + listener net.Listener + bindAddress string +} + +// startWebServer runs an internal web server to receive config details +func (s *authServer) Start() { + fs.Debug(nil, "Starting auth server on %s", s.bindAddress) + mux := http.NewServeMux() + server := &http.Server{ + Addr: s.bindAddress, + Handler: mux, + } + mux.HandleFunc("/favicon.ico", func(w http.ResponseWriter, req *http.Request) { + http.Error(w, "", 404) + return + }) + mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) { + fs.Debug(nil, "Received request on auth server") + code := req.FormValue("code") + if code != "" { + state := req.FormValue("state") + if state != s.state { + fs.Debug(nil, "State did not match: want %q got %q", s.state, state) + fmt.Fprintf(w, "

    Failure

    \n

    Auth state doesn't match

    ") + } else { + fs.Debug(nil, "Successfully got code") + fmt.Fprintf(w, "

    Success

    \n

    Cut and paste this code into rclone: %s

    ", code) + } + return + } + fs.Debug(nil, "No code found on request") + fmt.Fprintf(w, "

    Failed!

    \nNo code found.") + http.Error(w, "", 500) + }) + + var err error + s.listener, err = net.Listen("tcp", s.bindAddress) + if err != nil { + log.Fatalf("Failed to start auth webserver: %v", err) + } + server.Serve(s.listener) + fs.Debug(nil, "Closed auth server") +} + +func (s *authServer) Stop() { + fs.Debug(nil, "Closing auth server") + _ = s.listener.Close() +} diff --git a/rclone.go b/rclone.go index 35c93050b..1b5cc5715 100644 --- a/rclone.go +++ b/rclone.go @@ -16,6 +16,7 @@ import ( "github.com/ncw/rclone/fs" // Active file systems + _ "github.com/ncw/rclone/amazonclouddrive" _ "github.com/ncw/rclone/drive" _ "github.com/ncw/rclone/dropbox" _ "github.com/ncw/rclone/googlecloudstorage"