diff --git a/.gitignore b/.gitignore index e98cba6d2..339a7aad6 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ rclone rclonetest/rclonetest build docs/public -README.html -README.txt +MANUAL.md +MANUAL.html +MANUAL.txt rclone.1 diff --git a/Makefile b/Makefile index 289979fe2..e23560317 100644 --- a/Makefile +++ b/Makefile @@ -10,16 +10,19 @@ test: rclone go test ./... cd fs && ./test_all.sh -doc: rclone.1 README.html README.txt +doc: rclone.1 MANUAL.html MANUAL.txt -rclone.1: README.md - pandoc -s --from markdown --to man README.md -o rclone.1 +rclone.1: MANUAL.md + pandoc -s --from markdown --to man MANUAL.md -o rclone.1 -README.html: README.md - pandoc -s --from markdown_github --to html README.md -o README.html +MANUAL.md: make_manual.py docs/content/*.md + ./make_manual.py -README.txt: README.md - pandoc -s --from markdown_github --to plain README.md -o README.txt +MANUAL.html: MANUAL.md + pandoc -s --from markdown --to html MANUAL.md -o MANUAL.html + +MANUAL.txt: MANUAL.md + pandoc -s --from markdown --to plain MANUAL.md -o MANUAL.txt install: rclone install -d ${DESTDIR}/usr/bin @@ -29,7 +32,7 @@ clean: go clean ./... find . -name \*~ | xargs -r rm -f rm -rf build docs/public - rm -f rclone rclonetest/rclonetest rclone.1 README.html README.txt + rm -f rclone rclonetest/rclonetest rclone.1 MANUAL.md MANUAL.html MANUAL.txt website: cd docs && hugo @@ -52,11 +55,11 @@ tag: echo -e "package fs\n const Version = \"$(NEW_TAG)\"\n" | gofmt > fs/version.go perl -lpe 's/VERSION/${NEW_TAG}/g; s/DATE/'`date -I`'/g;' docs/content/downloads.md.in > docs/content/downloads.md git tag $(NEW_TAG) - @echo "Add this to changelog in README.md" + @echo "Add this to changelog in docs/content/changelog.md" @echo " * $(NEW_TAG) -" `date -I` @git log $(LAST_TAG)..$(NEW_TAG) --oneline @echo "Then commit the changes" - @echo git commit -m "Version $(NEW_TAG)" -a -v + @echo git commit -m \"Version $(NEW_TAG)\" -a -v @echo "And finally run make retag before make cross etc" retag: diff --git a/README.md b/README.md index e6329b3a9..552b3e2a2 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ -% rclone(1) User Manual -% Nick Craig-Wood -% Jul 7, 2014 - -Rclone -====== - [![Logo](http://rclone.org/img/rclone-120x120.png)](http://rclone.org/) +[Website](http://rclone.org) | +[Documentation](http://rclone.org/docs/) | +[Installation](http://rclone.org/install/) | +[G+](https://google.com/+RcloneOrg) + +[![Build Status](https://travis-ci.org/ncw/rclone.png)](https://travis-ci.org/ncw/rclone) [![GoDoc](https://godoc.org/github.com/ncw/rclone?status.svg)](https://godoc.org/github.com/ncw/rclone) + Rclone is a command line program to sync files and directories to and from * Google Drive @@ -26,358 +26,13 @@ Features * Check mode to check all MD5SUMs * Can sync to and from network, eg two different Drive accounts -See the Home page for more documentation and configuration walkthroughs. +See the home page for installation, usage, documentation, changelog +and configuration walkthroughs. * http://rclone.org/ -Install -------- - -Rclone is a Go program and comes as a single binary file. - -Download the binary for your OS from - - * http://rclone.org/downloads/ - -Or alternatively if you have Go installed use - - go install github.com/ncw/rclone - -and this will build the binary in `$GOPATH/bin`. - -Configure ---------- - -First you'll need to configure rclone. As the object storage systems -have quite complicated authentication these are kept in a config file -`.rclone.conf` in your home directory by default. (You can use the -`--config` option to choose a different config file.) - -The easiest way to make the config is to run rclone with the config -option, Eg - - rclone config - -Usage ------ - -Rclone syncs a directory tree from local to remote. - -Its basic syntax is - - Syntax: [options] subcommand - -See below for how to specify the source and destination paths. - -Subcommands ------------ - - rclone copy source:path dest:path - -Copy the source to the destination. Doesn't transfer -unchanged files, testing by size and modification time or -MD5SUM. Doesn't delete files from the destination. - - rclone sync source:path dest:path - -Sync the source to the destination, changing the destination -only. Doesn't transfer unchanged files, testing by size and -modification time or MD5SUM. Destination is updated to match -source, including deleting files if necessary. Since this can -cause data loss, test first with the `--dry-run` flag. - - rclone ls [remote:path] - -List all the objects in the the path with size and path. - - rclone lsd [remote:path] - -List all directories/containers/buckets in the the path. - - rclone lsl [remote:path] - -List all the objects in the the path with modification time, -size and path. - - rclone md5sum [remote:path] - -Produces an md5sum file for all the objects in the path. This -is in the same format as the standard md5sum tool produces. - - rclone mkdir remote:path - -Make the path if it doesn't already exist - - rclone rmdir remote:path - -Remove the path. Note that you can't remove a path with -objects in it, use purge for that. - - rclone purge remote:path - -Remove the path and all of its contents. - - rclone check source:path dest:path - -Checks the files in the source and destination match. It -compares sizes and MD5SUMs and prints a report of files which -don't match. It doesn't alter the source or destination. - - rclone config - -Enter an interactive configuration session. - - rclone help - -This help. - -General options: - -``` - --bwlimit=0: Bandwidth limit in kBytes/s, or use suffix k|M|G - --checkers=8: Number of checkers to run in parallel. - -c, --checksum=false: Skip based on checksum & size, not mod-time & size - --config="~/.rclone.conf": Config file. - --contimeout=1m0s: Connect timeout - -n, --dry-run=false: Do a trial run with no permanent changes - --log-file="": Log everything to this file - --modify-window=1ns: Max time diff to be considered the same - -q, --quiet=false: Print as little stuff as possible - --size-only=false: Skip based on size only, not mod-time or checksum - --stats=1m0s: Interval to print stats (0 to disable) - --timeout=5m0s: IO idle timeout - --transfers=4: Number of file transfers to run in parallel. - -v, --verbose=false: Print lots more stuff - -V, --version=false: Print the version number -``` - -Developer options: - -``` - --cpuprofile="": Write cpu profile to file -``` - -Local Filesystem ----------------- - -Paths are specified as normal filesystem paths, so - - rclone sync /home/source /tmp/destination - -Will sync `/home/source` to `/tmp/destination` - -Swift / Rackspace cloudfiles / Memset Memstore ----------------------------------------------- - -Paths are specified as remote:container (or remote: for the `lsd` -command.) You may put subdirectories in too, eg -`remote:container/path/to/dir`. - -So to copy a local directory to a swift container called backup: - - rclone sync /home/source swift:backup - -For more help see the [online docs on Openstack Swift](http://rclone.org/swift). - -Amazon S3 ---------- - -Paths are specified as remote:bucket. You may put subdirectories in -too, eg `remote:bucket/path/to/dir`. - -So to copy a local directory to a s3 container called backup - - rclone sync /home/source s3:backup - -For more help see the [online docs on Amazon S3](http://rclone.org/s3). - -Google drive ------------- - -Paths are specified as remote:path Drive paths may be as deep as required. - -The initial setup for drive involves getting a token from Google drive -which you need to do in your browser. `rclone config` walks you -through it. - -To copy a local directory to a drive directory called backup - - rclone copy /home/source remote:backup - -For more help see the [online docs on Google Drive](http://rclone.org/drive). - -Dropbox -------- - -Paths are specified as remote:path Dropbox paths may be as deep as required. - -The initial setup for dropbox involves getting a token from Dropbox -which you need to do in your browser. `rclone config` walks you -through it. - -To copy a local directory to a drive directory called backup - - rclone copy /home/source dropbox:backup - -For more help see the [online docs on Dropbox](http://rclone.org/dropbox). - -Google Cloud Storage --------------------- - -Paths are specified as remote:path Google Cloud Storage paths may be -as deep as required. - -The initial setup for Google Cloud Storage involves getting a token -from Google which you need to do in your browser. `rclone config` -walks you through it. - -To copy a local directory to a google cloud storage directory called backup - - rclone copy /home/source remote:backup - -For more help see the [online docs on Google Cloud Storage](http://rclone.org/googlecloudstorage/). - -Single file copies ------------------- - -Rclone can copy single files - - rclone src:path/to/file dst:path/dir - -Or - - rclone src:path/to/file dst:path/to/file - -Note that you can't rename the file if you are copying from one file to another. - License ------- This is free software under the terms of MIT the license (check the COPYING file included in this package). - -Bugs ----- - - * Empty directories left behind with Local and Drive - * eg purging a local directory with subdirectories doesn't work - -Changelog ---------- - * v1.14 - 2015-05-21 - * local: fix encoding of non utf-8 file names - fixes a duplicate file problem - * drive: docs about rate limiting - * google cloud storage: Fix compile after API change in "google.golang.org/api/storage/v1" - * v1.13 - 2015-05-10 - * Revise documentation (especially sync) - * Implement --timeout and --conntimeout - * s3: ignore etags from multipart uploads which aren't md5sums - * v1.12 - 2015-03-15 - * drive: Use chunked upload for files above a certain size - * drive: add --drive-chunk-size and --drive-upload-cutoff parameters - * drive: switch to insert from update when a failed copy deletes the upload - * core: Log duplicate files if they are detected - * v1.11 - 2015-03-04 - * swift: add region parameter - * drive: fix crash on failed to update remote mtime - * In remote paths, change native directory separators to / - * Add synchronization to ls/lsl/lsd output to stop corruptions - * Ensure all stats/log messages to go stderr - * Add --log-file flag to log everything (including panics) to file - * Make it possible to disable stats printing with --stats=0 - * Implement --bwlimit to limit data transfer bandwidth - * v1.10 - 2015-02-12 - * s3: list an unlimited number of items - * Fix getting stuck in the configurator - * v1.09 - 2015-02-07 - * windows: Stop drive letters (eg C:) getting mixed up with remotes (eg drive:) - * local: Fix directory separators on Windows - * drive: fix rate limit exceeded errors - * v1.08 - 2015-02-04 - * drive: fix subdirectory listing to not list entire drive - * drive: Fix SetModTime - * dropbox: adapt code to recent library changes - * v1.07 - 2014-12-23 - * google cloud storage: fix memory leak - * v1.06 - 2014-12-12 - * Fix "Couldn't find home directory" on OSX - * swift: Add tenant parameter - * Use new location of Google API packages - * v1.05 - 2014-08-09 - * Improved tests and consequently lots of minor fixes - * core: Fix race detected by go race detector - * core: Fixes after running errcheck - * drive: reset root directory on Rmdir and Purge - * fs: Document that Purger returns error on empty directory, test and fix - * google cloud storage: fix ListDir on subdirectory - * google cloud storage: re-read metadata in SetModTime - * s3: make reading metadata more reliable to work around eventual consistency problems - * s3: strip trailing / from ListDir() - * swift: return directories without / in ListDir - * v1.04 - 2014-07-21 - * google cloud storage: Fix crash on Update - * v1.03 - 2014-07-20 - * swift, s3, dropbox: fix updated files being marked as corrupted - * Make compile with go 1.1 again - * v1.02 - 2014-07-19 - * Implement Dropbox remote - * Implement Google Cloud Storage remote - * Verify Md5sums and Sizes after copies - * Remove times from "ls" command - lists sizes only - * Add add "lsl" - lists times and sizes - * Add "md5sum" command - * v1.01 - 2014-07-04 - * drive: fix transfer of big files using up lots of memory - * v1.00 - 2014-07-03 - * drive: fix whole second dates - * v0.99 - 2014-06-26 - * Fix --dry-run not working - * Make compatible with go 1.1 - * v0.98 - 2014-05-30 - * s3: Treat missing Content-Length as 0 for some ceph installations - * rclonetest: add file with a space in - * v0.97 - 2014-05-05 - * Implement copying of single files - * s3 & swift: support paths inside containers/buckets - * v0.96 - 2014-04-24 - * drive: Fix multiple files of same name being created - * drive: Use o.Update and fs.Put to optimise transfers - * Add version number, -V and --version - * v0.95 - 2014-03-28 - * rclone.org: website, docs and graphics - * drive: fix path parsing - * v0.94 - 2014-03-27 - * Change remote format one last time - * GNU style flags - * v0.93 - 2014-03-16 - * drive: store token in config file - * cross compile other versions - * set strict permissions on config file - * v0.92 - 2014-03-15 - * Config fixes and --config option - * v0.91 - 2014-03-15 - * Make config file - * v0.90 - 2013-06-27 - * Project named rclone - * v0.00 - 2012-11-18 - * Project started - - -Contact and support -------------------- - -The project website is at: - - * https://github.com/ncw/rclone - -There you can file bug reports, ask for help or send pull requests. - -Authors -------- - - * Nick Craig-Wood - -Contributors ------------- - - * Alex Couper diff --git a/cross-compile b/cross-compile index ffb35411a..1f8dbb454 100755 --- a/cross-compile +++ b/cross-compile @@ -21,8 +21,8 @@ mv build/rclone-${VERSION}-darwin-386 build/rclone-${VERSION}-osx-386 cd build for d in `ls`; do - cp -a ../README.txt $d/ - cp -a ../README.html $d/ + cp -a ../MANUAL.txt $d/README.txt + cp -a ../MANUAL.html $d/README.html cp -a ../rclone.1 $d/ zip -r9 $d.zip $d rm -rf $d diff --git a/docs/content/authors.md b/docs/content/authors.md new file mode 100644 index 000000000..cc65b60b9 --- /dev/null +++ b/docs/content/authors.md @@ -0,0 +1,15 @@ +--- +title: "Authors" +description: "Rclone Authors and Contributors" +date: "2014-06-16" +--- + +Authors +------- + + * Nick Craig-Wood + +Contributors +------------ + + * Alex Couper diff --git a/docs/content/bugs.md b/docs/content/bugs.md new file mode 100644 index 000000000..d576a711c --- /dev/null +++ b/docs/content/bugs.md @@ -0,0 +1,13 @@ +--- +title: "Bugs" +description: "Rclone Bugs" +date: "2014-06-16" +--- + +Bugs +---- + +Empty directories are left behind with Local and Drive + +eg purging a local directory with subdirectories doesn't work + diff --git a/docs/content/changelog.md b/docs/content/changelog.md new file mode 100644 index 000000000..24c376c60 --- /dev/null +++ b/docs/content/changelog.md @@ -0,0 +1,106 @@ +--- +title: "Documentation" +description: "Rclone Changelog" +date: "2015-06-06" +--- + +Changelog +--------- + + * v1.14 - 2015-05-21 + * local: fix encoding of non utf-8 file names - fixes a duplicate file problem + * drive: docs about rate limiting + * google cloud storage: Fix compile after API change in "google.golang.org/api/storage/v1" + * v1.13 - 2015-05-10 + * Revise documentation (especially sync) + * Implement --timeout and --conntimeout + * s3: ignore etags from multipart uploads which aren't md5sums + * v1.12 - 2015-03-15 + * drive: Use chunked upload for files above a certain size + * drive: add --drive-chunk-size and --drive-upload-cutoff parameters + * drive: switch to insert from update when a failed copy deletes the upload + * core: Log duplicate files if they are detected + * v1.11 - 2015-03-04 + * swift: add region parameter + * drive: fix crash on failed to update remote mtime + * In remote paths, change native directory separators to / + * Add synchronization to ls/lsl/lsd output to stop corruptions + * Ensure all stats/log messages to go stderr + * Add --log-file flag to log everything (including panics) to file + * Make it possible to disable stats printing with --stats=0 + * Implement --bwlimit to limit data transfer bandwidth + * v1.10 - 2015-02-12 + * s3: list an unlimited number of items + * Fix getting stuck in the configurator + * v1.09 - 2015-02-07 + * windows: Stop drive letters (eg C:) getting mixed up with remotes (eg drive:) + * local: Fix directory separators on Windows + * drive: fix rate limit exceeded errors + * v1.08 - 2015-02-04 + * drive: fix subdirectory listing to not list entire drive + * drive: Fix SetModTime + * dropbox: adapt code to recent library changes + * v1.07 - 2014-12-23 + * google cloud storage: fix memory leak + * v1.06 - 2014-12-12 + * Fix "Couldn't find home directory" on OSX + * swift: Add tenant parameter + * Use new location of Google API packages + * v1.05 - 2014-08-09 + * Improved tests and consequently lots of minor fixes + * core: Fix race detected by go race detector + * core: Fixes after running errcheck + * drive: reset root directory on Rmdir and Purge + * fs: Document that Purger returns error on empty directory, test and fix + * google cloud storage: fix ListDir on subdirectory + * google cloud storage: re-read metadata in SetModTime + * s3: make reading metadata more reliable to work around eventual consistency problems + * s3: strip trailing / from ListDir() + * swift: return directories without / in ListDir + * v1.04 - 2014-07-21 + * google cloud storage: Fix crash on Update + * v1.03 - 2014-07-20 + * swift, s3, dropbox: fix updated files being marked as corrupted + * Make compile with go 1.1 again + * v1.02 - 2014-07-19 + * Implement Dropbox remote + * Implement Google Cloud Storage remote + * Verify Md5sums and Sizes after copies + * Remove times from "ls" command - lists sizes only + * Add add "lsl" - lists times and sizes + * Add "md5sum" command + * v1.01 - 2014-07-04 + * drive: fix transfer of big files using up lots of memory + * v1.00 - 2014-07-03 + * drive: fix whole second dates + * v0.99 - 2014-06-26 + * Fix --dry-run not working + * Make compatible with go 1.1 + * v0.98 - 2014-05-30 + * s3: Treat missing Content-Length as 0 for some ceph installations + * rclonetest: add file with a space in + * v0.97 - 2014-05-05 + * Implement copying of single files + * s3 & swift: support paths inside containers/buckets + * v0.96 - 2014-04-24 + * drive: Fix multiple files of same name being created + * drive: Use o.Update and fs.Put to optimise transfers + * Add version number, -V and --version + * v0.95 - 2014-03-28 + * rclone.org: website, docs and graphics + * drive: fix path parsing + * v0.94 - 2014-03-27 + * Change remote format one last time + * GNU style flags + * v0.93 - 2014-03-16 + * drive: store token in config file + * cross compile other versions + * set strict permissions on config file + * v0.92 - 2014-03-15 + * Config fixes and --config option + * v0.91 - 2014-03-15 + * Make config file + * v0.90 - 2013-06-27 + * Project named rclone + * v0.00 - 2012-11-18 + * Project started diff --git a/docs/content/contact.md b/docs/content/contact.md index 9dcd1cf9a..470c93c5b 100644 --- a/docs/content/contact.md +++ b/docs/content/contact.md @@ -5,8 +5,17 @@ date: "2014-04-26" --- Contact the rclone project +-------------------------- + +The project website is at: + + * https://github.com/ncw/rclone + +There you can file bug reports, ask for help or contribute pull +requests. + +See also - * [Github project page for source, reporting bugs and pull requests](http://github.com/ncw/rclone) * Google+ page for general comments Or email [Nick Craig-Wood](mailto:nick@craig-wood.com) diff --git a/docs/content/docs.md b/docs/content/docs.md index 729228e33..7921cdce7 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -1,22 +1,9 @@ --- title: "Documentation" -description: "Rclone Documentation" -date: "2014-07-17" +description: "Rclone Usage" +date: "2015-06-06" --- -Install -------- - -Rclone is a Go program and comes as a single binary file. - -[Download](/downloads/) the relevant binary. - -Or alternatively if you have Go installed use - - go get github.com/ncw/rclone - -and this will build the binary in `$GOPATH/bin`. - Configure --------- @@ -30,11 +17,13 @@ option: rclone config -See below for detailed instructions for +See the following for detailed instructions for * [Google drive](/drive/) * [Amazon S3](/s3/) * [Swift / Rackspace Cloudfiles / Memset Memstore](/swift/) + * [Dropbox](/dropbox/) + * [Google Cloud Storage](/googlcloudstorage/) * [Local filesystem](/local/) Usage @@ -55,13 +44,13 @@ You can define as many storage paths as you like in the config file. Subcommands ----------- - rclone copy source:path dest:path +### rclone copy source:path dest:path ### Copy the source to the destination. Doesn't transfer unchanged files, testing by size and modification time or MD5SUM. Doesn't delete files from the destination. - rclone sync source:path dest:path +### rclone sync source:path dest:path ### Sync the source to the destination, changing the destination only. Doesn't transfer unchanged files, testing by size and @@ -69,102 +58,202 @@ modification time or MD5SUM. Destination is updated to match source, including deleting files if necessary. Since this can cause data loss, test first with the `--dry-run` flag. - rclone ls [remote:path] +### rclone ls [remote:path] ### List all the objects in the the path with size and path. - rclone lsd [remote:path] +### rclone lsd [remote:path] ### List all directories/containers/buckets in the the path. - rclone lsl [remote:path] +### rclone lsl [remote:path] ### List all the objects in the the path with modification time, size and path. - rclone md5sum [remote:path] +### rclone md5sum [remote:path] ### Produces an md5sum file for all the objects in the path. This is in the same format as the standard md5sum tool produces. - rclone mkdir remote:path +### rclone mkdir remote:path ### Make the path if it doesn't already exist - rclone rmdir remote:path +### rclone rmdir remote:path ### Remove the path. Note that you can't remove a path with objects in it, use purge for that. - rclone purge remote:path +### rclone purge remote:path ### Remove the path and all of its contents. - rclone check source:path dest:path +### rclone check source:path dest:path ### Checks the files in the source and destination match. It compares sizes and MD5SUMs and prints a report of files which don't match. It doesn't alter the source or destination. - rclone config +### rclone config ### Enter an interactive configuration session. - rclone help +### rclone help ### -This help. +Prints help on rclone commands and options. -``` - --bwlimit=0: Bandwidth limit in kBytes/s, or use suffix k|M|G - --checkers=8: Number of checkers to run in parallel. - -c, --checksum=false: Skip based on checksum, not mod-time & size - --config="~/.rclone.conf": Config file. - --contimeout=1m0s: Connect timeout - -n, --dry-run=false: Do a trial run with no permanent changes - --log-file="": Log everything to this file - --modify-window=1ns: Max time diff to be considered the same - -q, --quiet=false: Print as little stuff as possible - --size-only=false: Skip based on size only, not mod-time or checksum - --stats=1m0s: Interval to print stats (0 to disable) - --timeout=5m0s: IO idle timeout - --transfers=4: Number of file transfers to run in parallel. - -v, --verbose=false: Print lots more stuff - -V, --version=false: Print the version number -``` - -Developer options: - -``` - --cpuprofile="": Write cpu profile to file -``` - -License +Options ------- -This is free software under the terms of MIT the license (check the -COPYING file included in this package). +Rclone has a number of options to control its behaviour. -Bugs ----- +Options which use TIME use the go time parser. A duration string is a +possibly signed sequence of decimal numbers, each with optional +fraction and a unit suffix, such as "300ms", "-1.5h" or "2h45m". Valid +time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". - * Empty directories left behind with Local and Drive - * eg purging a local directory with subdirectories doesn't work +Options which use SIZE use kByte by default. However a suffix of `k` +for kBytes, `M` for MBytes and `G` for GBytes may be used. These are +the binary units, eg 2**10, 2**20, 2**30 respectively. -Contact and support -------------------- +### --bwlimit=SIZE ### -The project website is at: +Bandwidth limit in kBytes/s, or use suffix k|M|G. The default is `0` +which means to not limit bandwidth. - * https://github.com/ncw/rclone +For example to limit bandwidth usage to 10 MBytes/s use `--bwlimit 10M` -There you can file bug reports, ask for help or contribute patches. +This only limits the bandwidth of the data transfer, it doesn't limit +the bandwith of the directory listings etc. -Authors -------- +### --checkers=N ### - * Nick Craig-Wood +The number of checkers to run in parallel. Checkers do the equality +checking of files during a sync. For some storage systems (eg s3, +swift, dropbox) this can take a significant amount of time so they are +run in parallel. -Contributors ------------- +The default is to run 8 checkers in parallel. - * Alex Couper +### -c, --checksum ### + +Normally rclone will look at modification time and size of files to +see if they are equal. If you set this flag then rclone will check +MD5SUM and size to determine if files are equal. + +This is very useful when transferring between remotes which store the +MD5SUM on the object which include swift, s3, drive, and google cloud +storage. + +Eg `rclone --checksum sync s3:/bucket swift:/bucket` would run much +quicker than without the `--checksum` flag. + +When using this flag, rclone won't update mtimes of remote files if +they are incorrect as it would normally. + +### --config=CONFIG_FILE ### + +Specify the location of the rclone config file. Normally this is in +your home directory as a file called `.rclone.conf`. If you run +`rclone -h` and look at the help for the `--config` option you will +see where the default location is for you. Use this flag to override +the config location, eg `rclone --config=".myconfig" .config`. + +### --contimeout=TIME ### + +Set the connection timeout. This should be in go time format which +looks like `5s` for 5 seconds, `10m` for 10 minutes, or `3h30m`. + +The connection timeout is the amount of time rclone will wait for a +connection to go through to a remote object storage system. It is +`1m` by default. + +### -n, --dry-run ### + +Do a trial run with no permanent changes. Use this in combination +with the `-v` flag to see what rclone would do without actually doing +it. Useful when setting up the `sync` command. + +### --log-file=FILE ### + +Log all of rclone's output to FILE. This is not active by default. +This can be useful for tracking down problems with syncs in +combination with the `-v` flag. + +### --modify-window=TIME ### + +When checking whether a file has been modified, this is the maximum +allowed time difference that a file can have and still be considered +equivalent. + +The default is `1ns` unless this is overridden by a remote. For +example OS X only stores modification times to the nearest second so +if you are reading and writing to an OS X filing system this will be +`1s` by default. + +This command line flag allows you to override that computed default. + +### -q, --quiet ### + +Normally rclone outputs stats and a completion message. If you set +this flag it will make as little output as possible. + +### --size-only ### + +Normally rclone will look at modification time and size of files to +see if they are equal. If you set this flag then rclone will check +only the size. + +This can be useful transferring files from dropbox which have been +modified by the desktop sync client which doesn't set checksums of +modification times in the same way as rclone. + +When using this flag, rclone won't update mtimes of remote files if +they are incorrect as it would normally. + +### --stats=TIME ### + +Rclone will print stats at regular intervals to show its progress. + +This sets the interval. + +The default is `1m`. Use 0 to disable. + +### --timeout=TIME ### + +This sets the IO idle timeout. If a transfer has started but then +becomes idle for this long it is considered broken and disconnected. + +The default is `5m`. Set to 0 to disable. + +### --transfers=N ### + +The number of file transfers to run in parallel. It can sometimes be +useful to set this to a smaller number if the remote is giving a lot +of timeouts or bigger if you have lots of bandwidth and a fast remote. + +The default is to run 4 file transfers in parallel. + +### -v, --verbose ### + +If you set this flag, rclone will become very verbose telling you +about every file it considers and transfers. + +Very useful for debugging. + +### -V, --version ### + +Prints the version number + +Developer options +----------------- + +These options are useful when developing or debugging rclone. There +are also some more remote specific options which aren't documented +here which are used for testing. These start with remote name eg +`--drive-test-option`. + +### --cpuprofile=FILE ### + +Write cpu profile to file. This can be analysed with `go tool pprof`. diff --git a/docs/content/drive.md b/docs/content/drive.md index cf905ad49..61a64c8c9 100644 --- a/docs/content/drive.md +++ b/docs/content/drive.md @@ -69,13 +69,11 @@ To copy a local directory to a drive directory called backup rclone copy /home/source remote:backup -Modified time -------------- +### Modified time ### Google drive stores modification times accurate to 1 ms. -Revisions ---------- +### Revisions ### Google drive stores revisions of files. When you upload a change to an existing file to google drive using rclone it will create a new @@ -87,8 +85,7 @@ was * They are deleted after 30 days or 100 revisions (whatever comes first). * They do not count towards a user storage quota. -Limitations ------------ +### Limitations ### Drive has quite a lot of rate limiting. This causes rclone to be limited to transferring about 2 files per second only. Individual diff --git a/docs/content/dropbox.md b/docs/content/dropbox.md index b7016ee0f..0fadba83f 100644 --- a/docs/content/dropbox.md +++ b/docs/content/dropbox.md @@ -71,14 +71,12 @@ To copy a local directory to a dropbox directory called backup rclone copy /home/source remote:backup -Modified time -------------- +### Modified time ### Md5sums and timestamps in RFC3339 format accurate to 1ns are stored in a Dropbox datastore called "rclone". -Limitations ------------ +### Limitations ### Dropbox datastores are limited to 100,000 rows so this is the maximum number of files rclone can manage on Dropbox. diff --git a/docs/content/googlecloudstorage.md b/docs/content/googlecloudstorage.md index 7ff6a3170..6f15fe8d6 100644 --- a/docs/content/googlecloudstorage.md +++ b/docs/content/googlecloudstorage.md @@ -109,8 +109,7 @@ files in the bucket. rclone sync /home/local/directory remote:bucket -Modified time -------------- +### Modified time ### Google google cloud storage stores md5sums natively and rclone stores modification times as metadata on the object, under the "mtime" key in diff --git a/docs/content/install.md b/docs/content/install.md new file mode 100644 index 000000000..9512031ed --- /dev/null +++ b/docs/content/install.md @@ -0,0 +1,21 @@ +--- +title: "Install" +description: "Rclone Installation" +date: "2014-07-17" +--- + +Install +------- + +Rclone is a Go program and comes as a single binary file. + +[Download](/downloads/) the relevant binary. + +Or alternatively if you have Go installed use + + go get github.com/ncw/rclone + +and this will build the binary in `$GOPATH/bin`. + +See the [Usage section](/usage/) of the docs for how to use rclone, or +run `rclone -h`. diff --git a/docs/content/licence.md b/docs/content/licence.md new file mode 100644 index 000000000..2e4c1a978 --- /dev/null +++ b/docs/content/licence.md @@ -0,0 +1,34 @@ +--- +title: "Licence" +description: "Rclone Licence" +date: "2015-06-06" +--- + +License +------- + +This is free software under the terms of MIT the license (check the +COPYING file included with the source code). + +``` +Copyright (C) 2012 by Nick Craig-Wood http://www.craig-wood.com/nick/ + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +``` + diff --git a/docs/content/local.md b/docs/content/local.md index e20f046c0..c19399516 100644 --- a/docs/content/local.md +++ b/docs/content/local.md @@ -16,15 +16,13 @@ Will sync `/home/source` to `/tmp/destination` These can be configured into the config file for consistencies sake, but it is probably easier not to. -Modified time -------------- +### Modified time ### Rclone reads and writes the modified time using an accuracy determined by the OS. Typically this is 1ns on Linux, 10 ns on Windows and 1 Second on OS X. -Filenames ---------- +### Filenames ### Filenames are expected to be encoded in UTF-8 on disk. This is the normal case for Windows and OS X. There is a bit more uncertainty in @@ -33,7 +31,7 @@ names. If an invalid (non-UTF8) filename is read, the invalid caracters will be replaced with the unicode replacement character, '�'. `rclone` -will emit a warning in this case, eg +will emit a debug message in this case (use `-v` to see), eg ``` Local file system at .: Replacing invalid UTF-8 characters in "gro\xdf" diff --git a/docs/content/s3.md b/docs/content/s3.md index b53cfc444..c9c484c12 100644 --- a/docs/content/s3.md +++ b/docs/content/s3.md @@ -100,8 +100,7 @@ files in the bucket. rclone sync /home/local/directory remote:bucket -Modified time -------------- +### Modified time ### The modified time is stored as metadata on the object as `X-Amz-Meta-Mtime` as floating point since the epoch accurate to 1 ns. diff --git a/docs/content/swift.md b/docs/content/swift.md index 3314d5a0b..154f57262 100644 --- a/docs/content/swift.md +++ b/docs/content/swift.md @@ -87,8 +87,7 @@ excess files in the container. rclone sync /home/local/directory remote:container -Modified time -------------- +### Modified time ### The modified time is stored as metadata on the object as `X-Object-Meta-Mtime` as floating point since the epoch accurate to 1 diff --git a/docs/layouts/chrome/navbar.html b/docs/layouts/chrome/navbar.html index 8f71db423..39db25d11 100644 --- a/docs/layouts/chrome/navbar.html +++ b/docs/layouts/chrome/navbar.html @@ -12,8 +12,17 @@
diff --git a/make_manual.py b/make_manual.py new file mode 100755 index 000000000..8383809a8 --- /dev/null +++ b/make_manual.py @@ -0,0 +1,77 @@ +#!/usr/bin/python +""" +Make single page versions of the documentation for release and +conversion into man pages etc. +""" + +import os +import re +from datetime import datetime + +docpath = "docs/content" +outfile = "MANUAL.md" + +# Order to add docs segments to make outfile +docs = [ + "about.md", + "install.md", + "docs.md", + "drive.md", + "s3.md", + "swift.md", + "dropbox.md", + "googlecloudstorage.md", + "local.md", + "changelog.md", + "bugs.md", + "licence.md", + "authors.md", + "contact.md", +] + +# Docs which aren't made into outfile +ignore_docs = [ + "downloads.md", +] + +def read_doc(doc): + """Read file as a string""" + path = os.path.join(docpath, doc) + with open(path) as fd: + contents = fd.read() + parts = contents.split("---\n", 2) + if len(parts) != 3: + raise ValueError("Couldn't find --- markers: found %d parts" % len(parts)) + contents = parts[2].strip()+"\n\n" + # Remove icons + contents = re.sub(r'