fshttp: read config from ctx not passed in ConfigInfo #4685

backend/amazonclouddrive/amazonclouddrive.go

@@ -248,7 +248,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		return nil, err
 	}
 	root = parsePath(root)
-	baseClient := fshttp.NewClient(fs.GetConfig(ctx))
+	baseClient := fshttp.NewClient(ctx)
 	if do, ok := baseClient.Transport.(interface {
 		SetRequestFilter(f func(req *http.Request))
 	}); ok {
@@ -270,7 +270,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		ci:           ci,
 		c:            c,
 		pacer:        fs.NewPacer(ctx, pacer.NewAmazonCloudDrive(pacer.MinSleep(minSleep))),
-		noAuthClient: fshttp.NewClient(ci),
+		noAuthClient: fshttp.NewClient(ctx),
 	}
 	f.features = (&fs.Features{
 		CaseInsensitive:         true,

backend/azureblob/azureblob.go

@@ -417,7 +417,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		ci:          ci,
 		pacer:       fs.NewPacer(ctx, pacer.NewS3(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 		uploadToken: pacer.NewTokenDispenser(ci.Transfers),
-		client:      fshttp.NewClient(fs.GetConfig(ctx)),
+		client:      fshttp.NewClient(ctx),
 		cache:       bucket.NewCache(),
 		cntURLcache: make(map[string]*azblob.ContainerURL, 1),
 		pool: pool.New(

backend/b2/b2.go

@@ -421,7 +421,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		name:        name,
 		opt:         *opt,
 		ci:          ci,
-		srv:         rest.NewClient(fshttp.NewClient(fs.GetConfig(ctx))).SetErrorHandler(errorHandler),
+		srv:         rest.NewClient(fshttp.NewClient(ctx)).SetErrorHandler(errorHandler),
 		cache:       bucket.NewCache(),
 		_bucketID:   make(map[string]string, 1),
 		_bucketType: make(map[string]string, 1),

backend/box/box.go

@@ -169,7 +169,7 @@ func refreshJWTToken(ctx context.Context, jsonFile string, boxSubType string, na
 	}
 	signingHeaders := getSigningHeaders(boxConfig)
 	queryParams := getQueryParams(boxConfig)
-	client := fshttp.NewClient(fs.GetConfig(ctx))
+	client := fshttp.NewClient(ctx)
 	err = jwtutil.Config("box", name, claims, signingHeaders, queryParams, privateKey, m, client)
 	return err
 }
@@ -386,7 +386,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 
 	root = parsePath(root)
 
-	client := fshttp.NewClient(fs.GetConfig(ctx))
+	client := fshttp.NewClient(ctx)
 	var ts *oauthutil.TokenSource
 	// If not using an accessToken, create an oauth client and tokensource
 	if opt.AccessToken == "" {

backend/drive/drive.go

@@ -983,7 +983,7 @@ func configTeamDrive(ctx context.Context, opt *Options, m configmap.Mapper, name
 
 // getClient makes an http client according to the options
 func getClient(ctx context.Context, opt *Options) *http.Client {
-	t := fshttp.NewTransportCustom(fs.GetConfig(ctx), func(t *http.Transport) {
+	t := fshttp.NewTransportCustom(ctx, func(t *http.Transport) {
 		if opt.DisableHTTP2 {
 			t.TLSNextProto = map[string]func(string, *tls.Conn) http.RoundTripper{}
 		}

backend/fichier/fichier.go

@@ -195,7 +195,7 @@ func NewFs(ctx context.Context, name string, root string, config configmap.Mappe
 		CanHaveEmptyDirectories: true,
 	}).Fill(ctx, f)
 
-	client := fshttp.NewClient(fs.GetConfig(ctx))
+	client := fshttp.NewClient(ctx)
 
 	f.rest = rest.NewClient(client).SetRoot(apiBaseURL)
 

backend/filefabric/filefabric.go

@@ -425,7 +425,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 
 	root = parsePath(root)
 
-	client := fshttp.NewClient(fs.GetConfig(ctx))
+	client := fshttp.NewClient(ctx)
 
 	f := &Fs{
 		name:  name,

backend/googlecloudstorage/googlecloudstorage.go

@@ -375,7 +375,7 @@ func getServiceAccountClient(ctx context.Context, credentialsData []byte) (*http
 	if err != nil {
 		return nil, errors.Wrap(err, "error processing credentials")
 	}
-	ctxWithSpecialClient := oauthutil.Context(ctx, fshttp.NewClient(fs.GetConfig(ctx)))
+	ctxWithSpecialClient := oauthutil.Context(ctx, fshttp.NewClient(ctx))
 	return oauth2.NewClient(ctxWithSpecialClient, conf.TokenSource(ctxWithSpecialClient)), nil
 }
 

backend/googlephotos/googlephotos.go

@@ -254,7 +254,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		return nil, err
 	}
 
-	baseClient := fshttp.NewClient(fs.GetConfig(ctx))
+	baseClient := fshttp.NewClient(ctx)
 	oAuthClient, ts, err := oauthutil.NewClientWithBaseClient(ctx, name, m, oauthConfig, baseClient)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to configure Box")

backend/http/http.go

@@ -172,7 +172,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		return nil, err
 	}
 
-	client := fshttp.NewClient(fs.GetConfig(ctx))
+	client := fshttp.NewClient(ctx)
 
 	var isFile = false
 	if !strings.HasSuffix(u.String(), "/") {

backend/hubic/hubic.go

@@ -162,7 +162,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		Auth:           newAuth(f),
 		ConnectTimeout: 10 * ci.ConnectTimeout, // Use the timeouts in the transport
 		Timeout:        10 * ci.Timeout,        // Use the timeouts in the transport
-		Transport:      fshttp.NewTransport(fs.GetConfig(ctx)),
+		Transport:      fshttp.NewTransport(ctx),
 	}
 	err = c.Authenticate()
 	if err != nil {

backend/jottacloud/jottacloud.go

@@ -230,7 +230,7 @@ func shouldRetry(resp *http.Response, err error) (bool, error) {
 
 // v1config configure a jottacloud backend using legacy authentication
 func v1config(ctx context.Context, name string, m configmap.Mapper) {
-	srv := rest.NewClient(fshttp.NewClient(fs.GetConfig(ctx)))
+	srv := rest.NewClient(fshttp.NewClient(ctx))
 
 	fmt.Printf("\nDo you want to create a machine specific API key?\n\nRclone has it's own Jottacloud API KEY which works fine as long as one only uses rclone on a single machine. When you want to use rclone with this account on more than one machine it's recommended to create a machine specific API key. These keys can NOT be shared between machines.\n\n")
 	if config.Confirm(false) {
@@ -365,7 +365,7 @@ func doAuthV1(ctx context.Context, srv *rest.Client, username, password string)
 
 // v2config configure a jottacloud backend using the modern JottaCli token based authentication
 func v2config(ctx context.Context, name string, m configmap.Mapper) {
-	srv := rest.NewClient(fshttp.NewClient(fs.GetConfig(ctx)))
+	srv := rest.NewClient(fshttp.NewClient(ctx))
 
 	fmt.Printf("Generate a personal login token here: https://www.jottacloud.com/web/secure\n")
 	fmt.Printf("Login Token> ")
@@ -661,7 +661,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		return nil, errors.New("Outdated config - please reconfigure this backend")
 	}
 
-	baseClient := fshttp.NewClient(fs.GetConfig(ctx))
+	baseClient := fshttp.NewClient(ctx)
 
 	if ver == configVersion {
 		oauthConfig.ClientID = "jottacli"

backend/koofr/koofr.go

@@ -267,7 +267,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (ff fs.Fs
 		return nil, err
 	}
 	httpClient := httpclient.New()
-	httpClient.Client = fshttp.NewClient(fs.GetConfig(ctx))
+	httpClient.Client = fshttp.NewClient(ctx)
 	client := koofrclient.NewKoofrClientWithHTTPClient(opt.Endpoint, httpClient)
 	basicAuth := fmt.Sprintf("Basic %s",
 		base64.StdEncoding.EncodeToString([]byte(opt.User+":"+pass)))

backend/mailru/mailru.go

@@ -338,12 +338,12 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	}).Fill(ctx, f)
 
 	// Override few config settings and create a client
-	clientConfig := *fs.GetConfig(ctx)
+	newCtx, clientConfig := fs.AddConfig(ctx)
 	if opt.UserAgent != "" {
 		clientConfig.UserAgent = opt.UserAgent
 	}
 	clientConfig.NoGzip = true // Mimic official client, skip sending "Accept-Encoding: gzip"
-	f.cli = fshttp.NewClient(&clientConfig)
+	f.cli = fshttp.NewClient(newCtx)
 
 	f.srv = rest.NewClient(f.cli)
 	f.srv.SetRoot(api.APIServerURL)

backend/mega/mega.go

@@ -205,7 +205,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	defer megaCacheMu.Unlock()
 	srv := megaCache[opt.User]
 	if srv == nil {
-		srv = mega.New().SetClient(fshttp.NewClient(fs.GetConfig(ctx)))
+		srv = mega.New().SetClient(fshttp.NewClient(ctx))
 		srv.SetRetries(ci.LowLevelRetries) // let mega do the low level retries
 		srv.SetLogger(func(format string, v ...interface{}) {
 			fs.Infof("*go-mega*", format, v...)

backend/opendrive/opendrive.go

@@ -187,7 +187,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		name:  name,
 		root:  root,
 		opt:   *opt,
-		srv:   rest.NewClient(fshttp.NewClient(fs.GetConfig(ctx))).SetErrorHandler(errorHandler),
+		srv:   rest.NewClient(fshttp.NewClient(ctx)).SetErrorHandler(errorHandler),
 		pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 	}
 

backend/premiumizeme/premiumizeme.go

@@ -252,7 +252,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 			return nil, errors.Wrap(err, "failed to configure premiumize.me")
 		}
 	} else {
-		client = fshttp.NewClient(fs.GetConfig(ctx))
+		client = fshttp.NewClient(ctx)
 	}
 
 	f := &Fs{

backend/putio/fs.go

@@ -77,7 +77,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (f fs.Fs,
 		return nil, err
 	}
 	root = parsePath(root)
-	httpClient := fshttp.NewClient(fs.GetConfig(ctx))
+	httpClient := fshttp.NewClient(ctx)
 	oAuthClient, _, err := oauthutil.NewClientWithBaseClient(ctx, name, m, putioConfig, httpClient)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to configure putio")

backend/qingstor/qingstor.go

@@ -277,7 +277,7 @@ func qsServiceConnection(ctx context.Context, opt *Options) (*qs.Service, error)
 	cf.Host = host
 	cf.Port = port
 	// unsupported in v3.1: cf.ConnectionRetries = opt.ConnectionRetries
-	cf.Connection = fshttp.NewClient(fs.GetConfig(ctx))
+	cf.Connection = fshttp.NewClient(ctx)
 
 	return qs.Init(cf)
 }

backend/s3/s3.go

@@ -1405,7 +1405,7 @@ func (o *Object) split() (bucket, bucketPath string) {
 // getClient makes an http client according to the options
 func getClient(ctx context.Context, opt *Options) *http.Client {
 	// TODO: Do we need cookies too?
-	t := fshttp.NewTransportCustom(fs.GetConfig(ctx), func(t *http.Transport) {
+	t := fshttp.NewTransportCustom(ctx, func(t *http.Transport) {
 		if opt.DisableHTTP2 {
 			t.TLSNextProto = map[string]func(string, *tls.Conn) http.RoundTripper{}
 		}

backend/seafile/seafile.go

@@ -197,7 +197,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		opt:           *opt,
 		endpoint:      u,
 		endpointURL:   u.String(),
-		srv:           rest.NewClient(fshttp.NewClient(fs.GetConfig(ctx))).SetRoot(u.String()),
+		srv:           rest.NewClient(fshttp.NewClient(ctx)).SetRoot(u.String()),
 		pacer:         getPacer(ctx, opt.URL),
 	}
 	f.features = (&fs.Features{
@@ -343,7 +343,7 @@ func Config(ctx context.Context, name string, m configmap.Mapper) {
 	if !strings.HasPrefix(url, "/") {
 		url += "/"
 	}
-	srv := rest.NewClient(fshttp.NewClient(fs.GetConfig(ctx))).SetRoot(url)
+	srv := rest.NewClient(fshttp.NewClient(ctx)).SetRoot(url)
 
 	// We loop asking for a 2FA code
 	for {

backend/sftp/sftp.go

@@ -254,7 +254,7 @@ type Object struct {
 // convenience function that connects to the given network address,
 // initiates the SSH handshake, and then sets up a Client.
 func (f *Fs) dial(ctx context.Context, network, addr string, sshConfig *ssh.ClientConfig) (*ssh.Client, error) {
-	dialer := fshttp.NewDialer(fs.GetConfig(ctx))
+	dialer := fshttp.NewDialer(ctx)
 	conn, err := dialer.Dial(network, addr)
 	if err != nil {
 		return nil, err

backend/sugarsync/sugarsync.go

@@ -106,7 +106,7 @@ func init() {
 				Method: "POST",
 				Path:   "/app-authorization",
 			}
-			srv := rest.NewClient(fshttp.NewClient(fs.GetConfig(ctx))).SetRoot(rootURL) //  FIXME
+			srv := rest.NewClient(fshttp.NewClient(ctx)).SetRoot(rootURL) //  FIXME
 
 			// FIXME
 			//err = f.pacer.Call(func() (bool, error) {
@@ -403,7 +403,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	}
 
 	root = parsePath(root)
-	client := fshttp.NewClient(fs.GetConfig(ctx))
+	client := fshttp.NewClient(ctx)
 	f := &Fs{
 		name:       name,
 		root:       root,

backend/swift/swift.go

@@ -363,7 +363,7 @@ func swiftConnection(ctx context.Context, opt *Options, name string) (*swift.Con
 		EndpointType:                swift.EndpointType(opt.EndpointType),
 		ConnectTimeout:              10 * ci.ConnectTimeout, // Use the timeouts in the transport
 		Timeout:                     10 * ci.Timeout,        // Use the timeouts in the transport
-		Transport:                   fshttp.NewTransport(fs.GetConfig(ctx)),
+		Transport:                   fshttp.NewTransport(ctx),
 	}
 	if opt.EnvAuth {
 		err := c.ApplyEnvironment()

backend/webdav/odrvcookie/fetch.go

@@ -182,7 +182,7 @@ func (ca *CookieAuth) getSPToken(ctx context.Context) (conf *SharepointSuccessRe
 	}
 	req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext
 
-	client := fshttp.NewClient(fs.GetConfig(ctx))
+	client := fshttp.NewClient(ctx)
 	resp, err := client.Do(req)
 	if err != nil {
 		return nil, errors.Wrap(err, "Error while logging in to endpoint")

backend/webdav/webdav.go

@@ -336,7 +336,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		opt:         *opt,
 		endpoint:    u,
 		endpointURL: u.String(),
-		srv:         rest.NewClient(fshttp.NewClient(fs.GetConfig(ctx))).SetRoot(u.String()),
+		srv:         rest.NewClient(fshttp.NewClient(ctx)).SetRoot(u.String()),
 		pacer:       fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 		precision:   fs.ModTimeNotSupported,
 	}

cmd/rc/rc.go

@@ -177,7 +177,7 @@ func doCall(ctx context.Context, path string, in rc.Params) (out rc.Params, err
 	}
 
 	// Do HTTP request
-	client := fshttp.NewClient(fs.GetConfig(ctx))
+	client := fshttp.NewClient(ctx)
 	url += path
 	data, err := json.Marshal(in)
 	if err != nil {

fs/fshttp/http.go

@@ -35,13 +35,14 @@ var (
 
 // StartHTTPTokenBucket starts the token bucket if necessary
 func StartHTTPTokenBucket(ctx context.Context) {
-	if fs.GetConfig(ctx).TPSLimit > 0 {
+	ci := fs.GetConfig(ctx)
-		tpsBurst := fs.GetConfig(ctx).TPSLimitBurst
+	if ci.TPSLimit > 0 {
+		tpsBurst := ci.TPSLimitBurst
 		if tpsBurst < 1 {
 			tpsBurst = 1
 		}
-		tpsBucket = rate.NewLimiter(rate.Limit(fs.GetConfig(ctx).TPSLimit), tpsBurst)
+		tpsBucket = rate.NewLimiter(rate.Limit(ci.TPSLimit), tpsBurst)
-		fs.Infof(nil, "Starting HTTP transaction limiter: max %g transactions/s with burst %d", fs.GetConfig(ctx).TPSLimit, tpsBurst)
+		fs.Infof(nil, "Starting HTTP transaction limiter: max %g transactions/s with burst %d", ci.TPSLimit, tpsBurst)
 	}
 }
 
@@ -94,7 +95,7 @@ func (c *timeoutConn) Write(b []byte) (n int, err error) {
 
 // dial with context and timeouts
 func dialContextTimeout(ctx context.Context, network, address string, ci *fs.ConfigInfo) (net.Conn, error) {
-	dialer := NewDialer(ci)
+	dialer := NewDialer(ctx)
 	c, err := dialer.DialContext(ctx, network, address)
 	if err != nil {
 		return c, err
@@ -111,7 +112,8 @@ func ResetTransport() {
 // NewTransportCustom returns an http.RoundTripper with the correct timeouts.
 // The customize function is called if set to give the caller an opportunity to
 // customize any defaults in the Transport.
-func NewTransportCustom(ci *fs.ConfigInfo, customize func(*http.Transport)) http.RoundTripper {
+func NewTransportCustom(ctx context.Context, customize func(*http.Transport)) http.RoundTripper {
+	ci := fs.GetConfig(ctx)
 	// Start with a sensible set of defaults then override.
 	// This also means we get new stuff when it gets added to go
 	t := new(http.Transport)
@@ -178,17 +180,18 @@ func NewTransportCustom(ci *fs.ConfigInfo, customize func(*http.Transport)) http
 }
 
 // NewTransport returns an http.RoundTripper with the correct timeouts
-func NewTransport(ci *fs.ConfigInfo) http.RoundTripper {
+func NewTransport(ctx context.Context) http.RoundTripper {
 	(*noTransport).Do(func() {
-		transport = NewTransportCustom(ci, nil)
+		transport = NewTransportCustom(ctx, nil)
 	})
 	return transport
 }
 
 // NewClient returns an http.Client with the correct timeouts
-func NewClient(ci *fs.ConfigInfo) *http.Client {
+func NewClient(ctx context.Context) *http.Client {
+	ci := fs.GetConfig(ctx)
 	client := &http.Client{
-		Transport: NewTransport(ci),
+		Transport: NewTransport(ctx),
 	}
 	if ci.Cookie {
 		client.Jar = cookieJar
@@ -355,7 +358,8 @@ func (t *Transport) RoundTrip(req *http.Request) (resp *http.Response, err error
 
 // NewDialer creates a net.Dialer structure with Timeout, Keepalive
 // and LocalAddr set from rclone flags.
-func NewDialer(ci *fs.ConfigInfo) *net.Dialer {
+func NewDialer(ctx context.Context) *net.Dialer {
+	ci := fs.GetConfig(ctx)
 	dialer := &net.Dialer{
 		Timeout:   ci.ConnectTimeout,
 		KeepAlive: 30 * time.Second,

fs/operations/operations.go

@@ -1502,7 +1502,7 @@ type copyURLFunc func(ctx context.Context, dstFileName string, in io.ReadCloser,
 
 // copyURLFn copies the data from the url to the function supplied
 func copyURLFn(ctx context.Context, dstFileName string, url string, dstFileNameFromURL bool, fn copyURLFunc) (err error) {
-	client := fshttp.NewClient(fs.GetConfig(ctx))
+	client := fshttp.NewClient(ctx)
 	resp, err := client.Get(url)
 	if err != nil {
 		return err

lib/oauthutil/oauthutil.go

@@ -353,7 +353,7 @@ func NewClientWithBaseClient(ctx context.Context, name string, m configmap.Mappe
 // NewClient gets a token from the config file and configures a Client
 // with it.  It returns the client and a TokenSource which Invalidate may need to be called on
 func NewClient(ctx context.Context, name string, m configmap.Mapper, oauthConfig *oauth2.Config) (*http.Client, *TokenSource, error) {
-	return NewClientWithBaseClient(ctx, name, m, oauthConfig, fshttp.NewClient(fs.GetConfig(ctx)))
+	return NewClientWithBaseClient(ctx, name, m, oauthConfig, fshttp.NewClient(ctx))
 }
 
 // AuthResult is returned from the web server after authorization
@@ -526,7 +526,7 @@ version recommended):
 	}
 
 	// Exchange the code for a token
-	ctx = Context(ctx, fshttp.NewClient(fs.GetConfig(ctx)))
+	ctx = Context(ctx, fshttp.NewClient(ctx))
 	token, err := oauthConfig.Exchange(ctx, auth.Code)
 	if err != nil {
 		return errors.Wrap(err, "failed to get token")