From 3ed4a2e9634fb9f5644c5280636237f8e0b3c9f8 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Mon, 24 Jul 2023 23:23:46 +0100 Subject: [PATCH] sftp: stop uploads re-using the same ssh connection to improve performance Before this change we released the ssh connection back to the pool before the upload was finished. This meant that uploads were re-using the same ssh connection which reduces throughput. This releases the ssh connection back to the pool only after the upload has finished, or on error state. See: https://forum.rclone.org/t/sftp-backend-opens-less-connection-than-expected/40245 --- backend/sftp/sftp.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go index 8a9e0f3c3..62deb3d90 100644 --- a/backend/sftp/sftp.go +++ b/backend/sftp/sftp.go @@ -1992,9 +1992,10 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op if err != nil { return fmt.Errorf("Update: %w", err) } + // Hang on to the connection for the whole upload so it doesn't get re-used while we are uploading file, err := c.sftpClient.OpenFile(o.path(), os.O_WRONLY|os.O_CREATE|os.O_TRUNC) - o.fs.putSftpConnection(&c, err) if err != nil { + o.fs.putSftpConnection(&c, err) return fmt.Errorf("Update Create failed: %w", err) } // remove the file if upload failed @@ -2014,14 +2015,18 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op } _, err = file.ReadFrom(&sizeReader{Reader: in, size: src.Size()}) if err != nil { + o.fs.putSftpConnection(&c, err) remove() return fmt.Errorf("Update ReadFrom failed: %w", err) } err = file.Close() if err != nil { + o.fs.putSftpConnection(&c, err) remove() return fmt.Errorf("Update Close failed: %w", err) } + // Release connection only when upload has finished so we don't upload multiple files on the same connection + o.fs.putSftpConnection(&c, err) // Set the mod time - this stats the object if o.fs.opt.SetModTime == true err = o.SetModTime(ctx, src.ModTime(ctx))