From 44637dcd7fe1b4328cc9e6d73966f556c8747562 Mon Sep 17 00:00:00 2001 From: nielash Date: Fri, 10 Nov 2023 22:56:28 -0500 Subject: [PATCH] bisync: high-level retries if --resilient Before this change, bisync had no ability to retry in the event of sync errors. After this change, bisync will retry if --resilient is passed, but only in one direction at a time. We can safely retry in one direction because the source is still intact, even if the dest was left in a messy state. If the first direction still fails after our final retry, we abort and do NOT continue in the other direction, to prevent the messy dest from polluting the source. If the first direction succeeds, we do then allow retries in the other direction. The number of retries is controllable by --retries (default 3) bisync: high-level retries if --resilient Before this change, bisync had no ability to retry in the event of sync errors. After this change, bisync will retry if --resilient is passed, but only in one direction at a time. We can safely retry in one direction because the source is still intact, even if the dest was left in a messy state. If the first direction still fails after our final retry, we abort and do NOT continue in the other direction, to prevent the messy dest from polluting the source. If the first direction succeeds, we do then allow retries in the other direction. The number of retries is controllable by --retries (default 3) --- cmd/bisync/cmd.go | 3 +++ cmd/bisync/deltas.go | 8 ++++++++ cmd/bisync/queue.go | 11 +++++++++++ 3 files changed, 22 insertions(+) diff --git a/cmd/bisync/cmd.go b/cmd/bisync/cmd.go index c3a2e1e57..54a417cb4 100644 --- a/cmd/bisync/cmd.go +++ b/cmd/bisync/cmd.go @@ -46,6 +46,7 @@ type Options struct { IgnoreListingChecksum bool Resilient bool TestFn TestFunc // test-only option, for mocking errors + Retries int } // Default values @@ -103,6 +104,7 @@ func (x *CheckSyncMode) Type() string { var Opt Options func init() { + Opt.Retries = 3 cmd.Root.AddCommand(commandDefinition) cmdFlags := commandDefinition.Flags() flags.BoolVarP(cmdFlags, &Opt.Resync, "resync", "1", Opt.Resync, "Performs the resync run. Path1 files may overwrite Path2 versions. Consider using --verbose or --dry-run first.", "") @@ -118,6 +120,7 @@ func init() { flags.BoolVarP(cmdFlags, &Opt.NoCleanup, "no-cleanup", "", Opt.NoCleanup, "Retain working files (useful for troubleshooting and testing).", "") flags.BoolVarP(cmdFlags, &Opt.IgnoreListingChecksum, "ignore-listing-checksum", "", Opt.IgnoreListingChecksum, "Do not use checksums for listings (add --ignore-checksum to additionally skip post-copy checksum checks)", "") flags.BoolVarP(cmdFlags, &Opt.Resilient, "resilient", "", Opt.Resilient, "Allow future runs to retry after certain less-serious errors, instead of requiring --resync. Use at your own risk!", "") + flags.IntVarP(cmdFlags, &Opt.Retries, "retries", "", Opt.Retries, "Retry operations this many times if they fail", "") } // bisync command definition diff --git a/cmd/bisync/deltas.go b/cmd/bisync/deltas.go index 68fb538c3..770dac750 100644 --- a/cmd/bisync/deltas.go +++ b/cmd/bisync/deltas.go @@ -427,6 +427,10 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change changes1 = true b.indent("Path2", "Path1", "Do queued copies to") results2to1, err = b.fastCopy(ctx, b.fs2, b.fs1, copy2to1, "copy2to1") + + // retries, if any + results2to1, err = b.retryFastCopy(ctx, b.fs2, b.fs1, copy2to1, "copy2to1", results2to1, err) + if err != nil { return } @@ -439,6 +443,10 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change changes2 = true b.indent("Path1", "Path2", "Do queued copies to") results1to2, err = b.fastCopy(ctx, b.fs1, b.fs2, copy1to2, "copy1to2") + + // retries, if any + results1to2, err = b.retryFastCopy(ctx, b.fs1, b.fs2, copy1to2, "copy1to2", results1to2, err) + if err != nil { return } diff --git a/cmd/bisync/queue.go b/cmd/bisync/queue.go index f08a82f89..1c4a56b8e 100644 --- a/cmd/bisync/queue.go +++ b/cmd/bisync/queue.go @@ -14,6 +14,7 @@ import ( "github.com/rclone/rclone/fs/filter" "github.com/rclone/rclone/fs/operations" "github.com/rclone/rclone/fs/sync" + "github.com/rclone/rclone/lib/terminal" ) // Results represents a pair of synced files, as reported by the LoggerFn @@ -182,6 +183,16 @@ func (b *bisyncRun) fastCopy(ctx context.Context, fsrc, fdst fs.Fs, files bilib. return getResults, err } +func (b *bisyncRun) retryFastCopy(ctx context.Context, fsrc, fdst fs.Fs, files bilib.Names, queueName string, results []Results, err error) ([]Results, error) { + if err != nil && b.opt.Resilient && b.opt.Retries > 1 { + for tries := 1; tries <= b.opt.Retries; tries++ { + fs.Logf(queueName, Color(terminal.YellowFg, "Received error: %v - retrying as --resilient is set. Retry %d/%d"), err, tries, b.opt.Retries) + results, err = b.fastCopy(ctx, fsrc, fdst, files, queueName) + } + } + return results, err +} + func (b *bisyncRun) resyncDir(ctx context.Context, fsrc, fdst fs.Fs) ([]Results, error) { ignoreListingChecksum = b.opt.IgnoreListingChecksum logger.LoggerFn = WriteResults