dropbox: add --dropbox-batch-commit-timeout to control batch timeout

This also adds an Debug message log showing how long each batch took

See: #5491

backend/dropbox/batcher.go

@@ -147,8 +147,14 @@ func (b *batcher) finishBatchJobStatus(ctx context.Context, launchBatchStatus *f
 	}
 	var batchStatus *files.UploadSessionFinishBatchJobStatus
 	sleepTime := 100 * time.Millisecond
-	const maxTries = 120
+	const maxSleepTime = 1 * time.Second
-	for try := 1; try <= maxTries; try++ {
+	startTime := time.Now()
+	try := 1
+	for {
+		remaining := time.Duration(b.f.opt.BatchCommitTimeout) - time.Since(startTime)
+		if remaining < 0 {
+			break
+		}
 		err = b.f.pacer.Call(func() (bool, error) {
 			batchStatus, err = b.f.srv.UploadSessionFinishBatchCheck(&async.PollArg{
 				AsyncJobId: launchBatchStatus.AsyncJobId,
@@ -156,23 +162,25 @@ func (b *batcher) finishBatchJobStatus(ctx context.Context, launchBatchStatus *f
 			return shouldRetry(ctx, err)
 		})
 		if err != nil {
-			fs.Debugf(b.f, "Wait for batch: sleeping for %v after error: %v: try %d/%d", sleepTime, err, try, maxTries)
+			fs.Debugf(b.f, "Wait for batch: sleeping for %v after error: %v: try %d remaining %v", sleepTime, err, try, remaining)
 		} else {
 			if batchStatus.Tag == "complete" {
+				fs.Debugf(b.f, "Upload batch completed in %v", time.Since(startTime))
 				return batchStatus.Complete, nil
 			}
-			fs.Debugf(b.f, "Wait for batch: sleeping for %v after status: %q: try %d/%d", sleepTime, batchStatus.Tag, try, maxTries)
+			fs.Debugf(b.f, "Wait for batch: sleeping for %v after status: %q: try %d remaining %v", sleepTime, batchStatus.Tag, try, remaining)
 		}
 		time.Sleep(sleepTime)
 		sleepTime *= 2
-		if sleepTime > time.Second {
+		if sleepTime > maxSleepTime {
-			sleepTime = time.Second
+			sleepTime = maxSleepTime
 		}
+		try++
 	}
 	if err == nil {
 		err = errors.New("batch didn't complete")
 	}
-	return nil, errors.Wrapf(err, "wait for batch failed after %d tries", maxTries)
+	return nil, errors.Wrapf(err, "wait for batch failed after %d tries in %v", try, time.Since(startTime))
 }
 
 // commit a batch

backend/dropbox/dropbox.go

@@ -266,6 +266,11 @@ default based on the batch_mode in use.
 `,
 			Default:  fs.Duration(0),
 			Advanced: true,
+		}, {
+			Name:     "batch_commit_timeout",
+			Help:     `Max time to wait for a batch to finish comitting`,
+			Default:  fs.Duration(10 * time.Minute),
+			Advanced: true,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
@@ -285,15 +290,16 @@ default based on the batch_mode in use.
 
 // Options defines the configuration for this backend
 type Options struct {
-	ChunkSize     fs.SizeSuffix        `config:"chunk_size"`
+	ChunkSize          fs.SizeSuffix        `config:"chunk_size"`
-	Impersonate   string               `config:"impersonate"`
+	Impersonate        string               `config:"impersonate"`
-	SharedFiles   bool                 `config:"shared_files"`
+	SharedFiles        bool                 `config:"shared_files"`
-	SharedFolders bool                 `config:"shared_folders"`
+	SharedFolders      bool                 `config:"shared_folders"`
-	BatchMode     string               `config:"batch_mode"`
+	BatchMode          string               `config:"batch_mode"`
-	BatchSize     int                  `config:"batch_size"`
+	BatchSize          int                  `config:"batch_size"`
-	BatchTimeout  fs.Duration          `config:"batch_timeout"`
+	BatchTimeout       fs.Duration          `config:"batch_timeout"`
-	AsyncBatch    bool                 `config:"async_batch"`
+	BatchCommitTimeout fs.Duration          `config:"batch_commit_timeout"`
-	Enc           encoder.MultiEncoder `config:"encoding"`
+	AsyncBatch         bool                 `config:"async_batch"`
+	Enc                encoder.MultiEncoder `config:"encoding"`
 }
 
 // Fs represents a remote dropbox server