community
/
rclone
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

bisync: Add new --ignore-listing-checksum flag to distinguish from --ignore-checksum 

https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=6.%20%2D%2Dignore%2Dchecksum%20should%20be%20split%20into%20two%20flags%20for%20separate%20purposes
This commit is contained in:
nielash 2023-07-11 06:40:01 -04:00 committed by Nick Craig-Wood
parent 5ca61ab705
commit f01a50eb47
3 changed files with 52 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
cmd/bisync/cmd.go
View File

@ -27,18 +27,19 @@ import (
// Options keep bisync options // Options keep bisync options
type Options struct { type Options struct {
Resync bool Resync bool
CheckAccess bool CheckAccess bool
CheckFilename string CheckFilename string
CheckSync CheckSyncMode CheckSync CheckSyncMode
RemoveEmptyDirs bool RemoveEmptyDirs bool
MaxDelete int // percentage from 0 to 100 MaxDelete int // percentage from 0 to 100
Force bool Force bool
FiltersFile string FiltersFile string
Workdir string Workdir string
DryRun bool DryRun bool
NoCleanup bool NoCleanup bool
SaveQueues bool // save extra debugging files (test only flag) SaveQueues bool // save extra debugging files (test only flag)
IgnoreListingChecksum bool
} }
// Default values // Default values
@ -108,6 +109,7 @@ func init() {
flags.StringVarP(cmdFlags, &Opt.Workdir, "workdir", "", Opt.Workdir, makeHelp("Use custom working dir - useful for testing. (default: {WORKDIR})"), "") flags.StringVarP(cmdFlags, &Opt.Workdir, "workdir", "", Opt.Workdir, makeHelp("Use custom working dir - useful for testing. (default: {WORKDIR})"), "")
flags.BoolVarP(cmdFlags, &tzLocal, "localtime", "", tzLocal, "Use local time in listings (default: UTC)", "") flags.BoolVarP(cmdFlags, &tzLocal, "localtime", "", tzLocal, "Use local time in listings (default: UTC)", "")
flags.BoolVarP(cmdFlags, &Opt.NoCleanup, "no-cleanup", "", Opt.NoCleanup, "Retain working files (useful for troubleshooting and testing).", "") flags.BoolVarP(cmdFlags, &Opt.NoCleanup, "no-cleanup", "", Opt.NoCleanup, "Retain working files (useful for troubleshooting and testing).", "")
flags.BoolVarP(cmdFlags, &Opt.IgnoreListingChecksum, "ignore-listing-checksum", "", Opt.IgnoreListingChecksum, "Do not use checksums for listings (add --ignore-checksum to additionally skip post-copy checksum checks)", "")
} }
// bisync command definition // bisync command definition
@ -213,9 +215,9 @@ func (opt *Options) applyFilters(ctx context.Context) (context.Context, error) {
if opt.DryRun { if opt.DryRun {
fs.Infof(nil, "Skipped storing filters file hash to %s as --dry-run is set", hashFile) fs.Infof(nil, "Skipped storing filters file hash to %s as --dry-run is set", hashFile)
} else { } else {
fs.Infof(nil, "Storing filters file hash to %s", hashFile) fs.Infof(nil, "Storing filters file hash to %s", hashFile)
if err := os.WriteFile(hashFile, []byte(gotHash), bilib.PermSecure); err != nil { if err := os.WriteFile(hashFile, []byte(gotHash), bilib.PermSecure); err != nil {
return ctx, err return ctx, err
} }
} }
} }

5
cmd/bisync/listing.go
View File

@ -253,8 +253,9 @@ func (b *bisyncRun) makeListing(ctx context.Context, f fs.Fs, listing string) (l
ci := fs.GetConfig(ctx) ci := fs.GetConfig(ctx)
depth := ci.MaxDepth depth := ci.MaxDepth
hashType := hash.None hashType := hash.None
if !ci.IgnoreChecksum { if !b.opt.IgnoreListingChecksum {
// Currently bisync just honors --ignore-checksum // Currently bisync just honors --ignore-listing-checksum
// (note that this is different from --ignore-checksum)
// TODO add full support for checksums and related flags // TODO add full support for checksums and related flags
hashType = f.Hashes().GetOne() hashType = f.Hashes().GetOne()
} }

32
docs/content/bisync.md
View File

@ -95,6 +95,8 @@ Optional Flags:
-1, --resync Performs the resync run. -1, --resync Performs the resync run.
Warning: Path1 files may overwrite Path2 versions. Warning: Path1 files may overwrite Path2 versions.
Consider using `--verbose` or `--dry-run` first. Consider using `--verbose` or `--dry-run` first.
--ignore-listing-checksum Do not use checksums for listings
(add --ignore-checksum to additionally skip post-copy checksum checks)
--localtime Use local time in listings (default: UTC) --localtime Use local time in listings (default: UTC)
--no-cleanup Retain working files (useful for troubleshooting and testing). --no-cleanup Retain working files (useful for troubleshooting and testing).
--workdir PATH Use custom working directory (useful for testing). --workdir PATH Use custom working directory (useful for testing).
@ -255,6 +257,34 @@ sync run times for very large numbers of files.
The check may be run manually with `--check-sync=only`. It runs only the The check may be run manually with `--check-sync=only`. It runs only the
integrity check and terminates without actually synching. integrity check and terminates without actually synching.
#### --ignore-listing-checksum
By default, bisync will retrieve (or generate) checksums (for backends that support them)
when creating the listings for both paths, and store the checksums in the listing files.
`--ignore-listing-checksum` will disable this behavior, which may speed things up considerably,
especially on backends (such as [local](/local/)) where hashes must be computed on the fly instead of retrieved.
Please note the following:
* While checksums are (by default) generated and stored in the listing files,
they are NOT currently used for determining diffs (deltas).
It is anticipated that full checksum support will be added in a future version.
* `--ignore-listing-checksum` is NOT the same as [`--ignore-checksum`](/docs/#ignore-checksum),
and you may wish to use one or the other, or both. In a nutshell:
`--ignore-listing-checksum` controls whether checksums are considered when scanning for diffs,
while `--ignore-checksum` controls whether checksums are considered during the copy/sync operations that follow,
if there ARE diffs.
* Unless `--ignore-listing-checksum` is passed, bisync currently computes hashes for one path
*even when there's no common hash with the other path*
(for example, a [crypt](/crypt/#modified-time-and-hashes) remote.)
* If both paths support checksums and have a common hash,
AND `--ignore-listing-checksum` was not specified when creating the listings,
`--check-sync=only` can be used to compare Path1 vs. Path2 checksums (as of the time the previous listings were created.)
However, `--check-sync=only` will NOT include checksums if the previous listings
were generated on a run using `--ignore-listing-checksum`. For a more robust integrity check of the current state,
consider using [`check`](commands/rclone_check/)
(or [`cryptcheck`](/commands/rclone_cryptcheck/), if at least one path is a `crypt` remote.)
## Operation ## Operation
### Runtime flow details ### Runtime flow details
@ -1134,3 +1164,5 @@ causing dry runs to inadvertently commit filter changes
causing bisync to consider more files than necessary due to overbroad filters during delete operations causing bisync to consider more files than necessary due to overbroad filters during delete operations
* [Improved detection of false positive change conflicts](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=1.%20Identical%20files%20should%20be%20left%20alone%2C%20even%20if%20new/newer/changed%20on%20both%20sides) * [Improved detection of false positive change conflicts](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=1.%20Identical%20files%20should%20be%20left%20alone%2C%20even%20if%20new/newer/changed%20on%20both%20sides)
(identical files are now left alone instead of renamed) (identical files are now left alone instead of renamed)
* Added [new `--ignore-listing-checksum` flag](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=6.%20%2D%2Dignore%2Dchecksum%20should%20be%20split%20into%20two%20flags%20for%20separate%20purposes)
to distinguish from `--ignore-checksum`