diff --git a/backend/s3/s3.go b/backend/s3/s3.go
index e62d85ee7..2e1f1d574 100644
--- a/backend/s3/s3.go
+++ b/backend/s3/s3.go
@@ -1099,7 +1099,7 @@ func s3Connection(opt *Options) (*s3.S3, *session.Session, error) {
 		opt.ForcePathStyle = false
 	}
 	awsConfig := aws.NewConfig().
-		WithMaxRetries(fs.Config.LowLevelRetries).
+		WithMaxRetries(0). // Rely on rclone's retry logic
 		WithCredentials(cred).
 		WithHTTPClient(fshttp.NewClient(fs.Config)).
 		WithS3ForcePathStyle(opt.ForcePathStyle).
@@ -1206,17 +1206,12 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		return nil, err
 	}
 
-	pc := fs.NewPacer(pacer.NewS3(pacer.MinSleep(minSleep)))
-	// Set pacer retries to 0 because we are relying on SDK retry mechanism.
-	// Setting it to 1 because in context of pacer it means 1 attempt.
-	pc.SetRetries(1)
-
 	f := &Fs{
 		name:  name,
 		opt:   *opt,
 		c:     c,
 		ses:   ses,
-		pacer: pc,
+		pacer: fs.NewPacer(pacer.NewS3(pacer.MinSleep(minSleep))),
 		cache: bucket.NewCache(),
 		srv:   fshttp.NewClient(fs.Config),
 		pools: make(map[int64]*pool.Pool),