From 66c23723e3deb2a20bb82c9ffd2a101ceed18687 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 4 Sep 2019 20:21:10 +0100 Subject: [PATCH] Add context to all http.NewRequest #3257 When we drop support for go1.12 we can use http.NewRequestWithContext --- backend/drive/drive.go | 23 ++++++++++--------- backend/drive/upload.go | 21 +++++++++-------- .../googlecloudstorage/googlecloudstorage.go | 1 + backend/http/http.go | 15 ++++++++---- backend/hubic/auth.go | 3 ++- backend/hubic/hubic.go | 4 +++- backend/putio/fs.go | 6 +++-- backend/putio/object.go | 6 ++++- backend/s3/s3.go | 2 +- backend/webdav/odrvcookie/fetch.go | 8 ++++--- backend/webdav/webdav.go | 8 +++---- cmd/rc/rc.go | 16 +++++++------ lib/rest/rest.go | 3 ++- 13 files changed, 70 insertions(+), 46 deletions(-) diff --git a/backend/drive/drive.go b/backend/drive/drive.go index dc0605483..ec535e22e 100644 --- a/backend/drive/drive.go +++ b/backend/drive/drive.go @@ -1734,7 +1734,7 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo, } } else { // Upload the file in chunks - info, err = f.Upload(in, size, srcMimeType, "", remote, createInfo) + info, err = f.Upload(ctx, in, size, srcMimeType, "", remote, createInfo) if err != nil { return nil, err } @@ -2499,7 +2499,7 @@ func (o *baseObject) Storable() bool { // httpResponse gets an http.Response object for the object // using the url and method passed in -func (o *baseObject) httpResponse(url, method string, options []fs.OpenOption) (req *http.Request, res *http.Response, err error) { +func (o *baseObject) httpResponse(ctx context.Context, url, method string, options []fs.OpenOption) (req *http.Request, res *http.Response, err error) { if url == "" { return nil, nil, errors.New("forbidden to download - check sharing permission") } @@ -2507,6 +2507,7 @@ func (o *baseObject) httpResponse(url, method string, options []fs.OpenOption) ( if err != nil { return req, nil, err } + req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext fs.OpenOptionAddHTTPHeaders(req.Header, options) if o.bytes == 0 { // Don't supply range requests for 0 length objects as they always fail @@ -2577,8 +2578,8 @@ func isGoogleError(err error, what string) bool { } // open a url for reading -func (o *baseObject) open(url string, options ...fs.OpenOption) (in io.ReadCloser, err error) { - _, res, err := o.httpResponse(url, "GET", options) +func (o *baseObject) open(ctx context.Context, url string, options ...fs.OpenOption) (in io.ReadCloser, err error) { + _, res, err := o.httpResponse(ctx, url, "GET", options) if err != nil { if isGoogleError(err, "cannotDownloadAbusiveFile") { if o.fs.opt.AcknowledgeAbuse { @@ -2589,7 +2590,7 @@ func (o *baseObject) open(url string, options ...fs.OpenOption) (in io.ReadClose url += "?" } url += "acknowledgeAbuse=true" - _, res, err = o.httpResponse(url, "GET", options) + _, res, err = o.httpResponse(ctx, url, "GET", options) } else { err = errors.Wrap(err, "Use the --drive-acknowledge-abuse flag to download this file") } @@ -2618,7 +2619,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read o.v2Download = false } } - return o.baseObject.open(o.url, options...) + return o.baseObject.open(ctx, o.url, options...) } func (o *documentObject) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) { // Update the size with what we are reading as it can change from @@ -2643,7 +2644,7 @@ func (o *documentObject) Open(ctx context.Context, options ...fs.OpenOption) (in if offset != 0 { return nil, errors.New("partial downloads are not supported while exporting Google Documents") } - in, err = o.baseObject.open(o.url, options...) + in, err = o.baseObject.open(ctx, o.url, options...) if in != nil { in = &openDocumentFile{o: o, in: in} } @@ -2678,7 +2679,7 @@ func (o *linkObject) Open(ctx context.Context, options ...fs.OpenOption) (in io. return ioutil.NopCloser(bytes.NewReader(data)), nil } -func (o *baseObject) update(updateInfo *drive.File, uploadMimeType string, in io.Reader, +func (o *baseObject) update(ctx context.Context, updateInfo *drive.File, uploadMimeType string, in io.Reader, src fs.ObjectInfo) (info *drive.File, err error) { // Make the API request to upload metadata and file data. size := src.Size() @@ -2696,7 +2697,7 @@ func (o *baseObject) update(updateInfo *drive.File, uploadMimeType string, in io return } // Upload the file in chunks - return o.fs.Upload(in, size, uploadMimeType, o.id, o.remote, updateInfo) + return o.fs.Upload(ctx, in, size, uploadMimeType, o.id, o.remote, updateInfo) } // Update the already existing object @@ -2710,7 +2711,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op MimeType: srcMimeType, ModifiedTime: src.ModTime(ctx).Format(timeFormatOut), } - info, err := o.baseObject.update(updateInfo, srcMimeType, in, src) + info, err := o.baseObject.update(ctx, updateInfo, srcMimeType, in, src) if err != nil { return err } @@ -2747,7 +2748,7 @@ func (o *documentObject) Update(ctx context.Context, in io.Reader, src fs.Object } updateInfo.MimeType = importMimeType - info, err := o.baseObject.update(updateInfo, srcMimeType, in, src) + info, err := o.baseObject.update(ctx, updateInfo, srcMimeType, in, src) if err != nil { return err } diff --git a/backend/drive/upload.go b/backend/drive/upload.go index 75e88779a..813a9b08b 100644 --- a/backend/drive/upload.go +++ b/backend/drive/upload.go @@ -11,6 +11,7 @@ package drive import ( + "context" "encoding/json" "fmt" "io" @@ -50,7 +51,7 @@ type resumableUpload struct { } // Upload the io.Reader in of size bytes with contentType and info -func (f *Fs) Upload(in io.Reader, size int64, contentType, fileID, remote string, info *drive.File) (*drive.File, error) { +func (f *Fs) Upload(ctx context.Context, in io.Reader, size int64, contentType, fileID, remote string, info *drive.File) (*drive.File, error) { params := url.Values{ "alt": {"json"}, "uploadType": {"resumable"}, @@ -81,6 +82,7 @@ func (f *Fs) Upload(in io.Reader, size int64, contentType, fileID, remote string if err != nil { return false, err } + req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext googleapi.Expand(req.URL, map[string]string{ "fileId": fileID, }) @@ -106,12 +108,13 @@ func (f *Fs) Upload(in io.Reader, size int64, contentType, fileID, remote string MediaType: contentType, ContentLength: size, } - return rx.Upload() + return rx.Upload(ctx) } // Make an http.Request for the range passed in -func (rx *resumableUpload) makeRequest(start int64, body io.ReadSeeker, reqSize int64) *http.Request { +func (rx *resumableUpload) makeRequest(ctx context.Context, start int64, body io.ReadSeeker, reqSize int64) *http.Request { req, _ := http.NewRequest("POST", rx.URI, body) + req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext req.ContentLength = reqSize if reqSize != 0 { req.Header.Set("Content-Range", fmt.Sprintf("bytes %v-%v/%v", start, start+reqSize-1, rx.ContentLength)) @@ -129,8 +132,8 @@ var rangeRE = regexp.MustCompile(`^0\-(\d+)$`) // Query drive for the amount transferred so far // // If error is nil, then start should be valid -func (rx *resumableUpload) transferStatus() (start int64, err error) { - req := rx.makeRequest(0, nil, 0) +func (rx *resumableUpload) transferStatus(ctx context.Context) (start int64, err error) { + req := rx.makeRequest(ctx, 0, nil, 0) res, err := rx.f.client.Do(req) if err != nil { return 0, err @@ -157,9 +160,9 @@ func (rx *resumableUpload) transferStatus() (start int64, err error) { } // Transfer a chunk - caller must call googleapi.CloseBody(res) if err == nil || res != nil -func (rx *resumableUpload) transferChunk(start int64, chunk io.ReadSeeker, chunkSize int64) (int, error) { +func (rx *resumableUpload) transferChunk(ctx context.Context, start int64, chunk io.ReadSeeker, chunkSize int64) (int, error) { _, _ = chunk.Seek(0, io.SeekStart) - req := rx.makeRequest(start, chunk, chunkSize) + req := rx.makeRequest(ctx, start, chunk, chunkSize) res, err := rx.f.client.Do(req) if err != nil { return 599, err @@ -192,7 +195,7 @@ func (rx *resumableUpload) transferChunk(start int64, chunk io.ReadSeeker, chunk // Upload uploads the chunks from the input // It retries each chunk using the pacer and --low-level-retries -func (rx *resumableUpload) Upload() (*drive.File, error) { +func (rx *resumableUpload) Upload(ctx context.Context) (*drive.File, error) { start := int64(0) var StatusCode int var err error @@ -207,7 +210,7 @@ func (rx *resumableUpload) Upload() (*drive.File, error) { // Transfer the chunk err = rx.f.pacer.Call(func() (bool, error) { fs.Debugf(rx.remote, "Sending chunk %d length %d", start, reqSize) - StatusCode, err = rx.transferChunk(start, chunk, reqSize) + StatusCode, err = rx.transferChunk(ctx, start, chunk, reqSize) again, err := shouldRetry(err) if StatusCode == statusResumeIncomplete || StatusCode == http.StatusCreated || StatusCode == http.StatusOK { again = false diff --git a/backend/googlecloudstorage/googlecloudstorage.go b/backend/googlecloudstorage/googlecloudstorage.go index cd6224225..d7379566f 100644 --- a/backend/googlecloudstorage/googlecloudstorage.go +++ b/backend/googlecloudstorage/googlecloudstorage.go @@ -1007,6 +1007,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read if err != nil { return nil, err } + req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext fs.FixRangeOption(options, o.bytes) fs.OpenOptionAddHTTPHeaders(req.Header, options) var res *http.Response diff --git a/backend/http/http.go b/backend/http/http.go index 997fee42a..b6e73e4e6 100644 --- a/backend/http/http.go +++ b/backend/http/http.go @@ -124,6 +124,7 @@ func statusError(res *http.Response, err error) error { // NewFs creates a new Fs object from the name and root. It connects to // the host specified in the config file. func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { + ctx := context.TODO() // Parse config into Options struct opt := new(Options) err := configstruct.Set(m, opt) @@ -162,6 +163,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { // check to see if points to a file req, err := http.NewRequest("HEAD", u.String(), nil) if err == nil { + req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext addHeaders(req, opt) res, err := noRedir.Do(req) err = statusError(res, err) @@ -237,7 +239,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { fs: f, remote: remote, } - err := o.stat() + err := o.stat(ctx) if err != nil { return nil, err } @@ -355,7 +357,7 @@ func (f *Fs) addHeaders(req *http.Request) { } // Read the directory passed in -func (f *Fs) readDir(dir string) (names []string, err error) { +func (f *Fs) readDir(ctx context.Context, dir string) (names []string, err error) { URL := f.url(dir) u, err := url.Parse(URL) if err != nil { @@ -369,6 +371,7 @@ func (f *Fs) readDir(dir string) (names []string, err error) { if err != nil { return nil, errors.Wrap(err, "readDir failed") } + req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext f.addHeaders(req) res, err := f.httpClient.Do(req) if err == nil { @@ -408,7 +411,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e if !strings.HasSuffix(dir, "/") && dir != "" { dir += "/" } - names, err := f.readDir(dir) + names, err := f.readDir(ctx, dir) if err != nil { return nil, errors.Wrapf(err, "error listing %q", dir) } @@ -424,7 +427,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e fs: f, remote: remote, } - switch err = file.stat(); err { + switch err = file.stat(ctx); err { case nil: entries = append(entries, file) case fs.ErrorNotAFile: @@ -492,12 +495,13 @@ func (o *Object) url() string { } // stat updates the info field in the Object -func (o *Object) stat() error { +func (o *Object) stat(ctx context.Context) error { url := o.url() req, err := http.NewRequest("HEAD", url, nil) if err != nil { return errors.Wrap(err, "stat failed") } + req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext o.fs.addHeaders(req) res, err := o.fs.httpClient.Do(req) if err == nil && res.StatusCode == http.StatusNotFound { @@ -546,6 +550,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read if err != nil { return nil, errors.Wrap(err, "Open failed") } + req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext // Add optional headers for k, v := range fs.OpenOptionHeaders(options) { diff --git a/backend/hubic/auth.go b/backend/hubic/auth.go index d38a5f2b4..62213c35f 100644 --- a/backend/hubic/auth.go +++ b/backend/hubic/auth.go @@ -1,6 +1,7 @@ package hubic import ( + "context" "net/http" "time" @@ -26,7 +27,7 @@ func newAuth(f *Fs) *auth { func (a *auth) Request(*swift.Connection) (r *http.Request, err error) { const retries = 10 for try := 1; try <= retries; try++ { - err = a.f.getCredentials() + err = a.f.getCredentials(context.TODO()) if err == nil { break } diff --git a/backend/hubic/hubic.go b/backend/hubic/hubic.go index e86448986..57b72070b 100644 --- a/backend/hubic/hubic.go +++ b/backend/hubic/hubic.go @@ -7,6 +7,7 @@ package hubic // to be revisted after some actual experience. import ( + "context" "encoding/json" "fmt" "io/ioutil" @@ -115,11 +116,12 @@ func (f *Fs) String() string { // getCredentials reads the OpenStack Credentials using the Hubic API // // The credentials are read into the Fs -func (f *Fs) getCredentials() (err error) { +func (f *Fs) getCredentials(ctx context.Context) (err error) { req, err := http.NewRequest("GET", "https://api.hubic.com/1.0/account/credentials", nil) if err != nil { return err } + req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext resp, err := f.client.Do(req) if err != nil { return err diff --git a/backend/putio/fs.go b/backend/putio/fs.go index 31698c611..4139a4746 100644 --- a/backend/putio/fs.go +++ b/backend/putio/fs.go @@ -289,6 +289,7 @@ func (f *Fs) createUpload(ctx context.Context, name string, size int64, parentID if err != nil { return false, err } + req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext req.Header.Set("tus-resumable", "1.0.0") req.Header.Set("upload-length", strconv.FormatInt(size, 10)) b64name := base64.StdEncoding.EncodeToString([]byte(name)) @@ -354,7 +355,7 @@ func (f *Fs) sendUpload(ctx context.Context, location string, size int64, in io. func (f *Fs) transferChunk(ctx context.Context, location string, start int64, chunk io.ReadSeeker, chunkSize int64) (fileID int64, err error) { // defer log.Trace(f, "location=%v, start=%v, chunkSize=%v", location, start, chunkSize)("fileID=%v, err=%v", fileID, &err) _, _ = chunk.Seek(0, io.SeekStart) - req, err := f.makeUploadPatchRequest(location, chunk, start, chunkSize) + req, err := f.makeUploadPatchRequest(ctx, location, chunk, start, chunkSize) if err != nil { return 0, err } @@ -379,11 +380,12 @@ func (f *Fs) transferChunk(ctx context.Context, location string, start int64, ch return fileID, nil } -func (f *Fs) makeUploadPatchRequest(location string, in io.Reader, offset, length int64) (*http.Request, error) { +func (f *Fs) makeUploadPatchRequest(ctx context.Context, location string, in io.Reader, offset, length int64) (*http.Request, error) { req, err := http.NewRequest("PATCH", location, in) if err != nil { return nil, err } + req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext req.Header.Set("tus-resumable", "1.0.0") req.Header.Set("upload-offset", strconv.FormatInt(offset, 10)) req.Header.Set("content-length", strconv.FormatInt(length, 10)) diff --git a/backend/putio/object.go b/backend/putio/object.go index 2ffe24b37..550557b55 100644 --- a/backend/putio/object.go +++ b/backend/putio/object.go @@ -223,7 +223,11 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read var resp *http.Response headers := fs.OpenOptionHeaders(options) err = o.fs.pacer.Call(func() (bool, error) { - req, _ := http.NewRequest(http.MethodGet, storageURL, nil) + req, err := http.NewRequest(http.MethodGet, storageURL, nil) + if err != nil { + return shouldRetry(err) + } + req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext req.Header.Set("User-Agent", o.fs.client.UserAgent) // merge headers with extra headers diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 5fdc731d5..e09bbf029 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -1942,7 +1942,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op if err != nil { return errors.Wrap(err, "s3 upload: new request") } - httpReq = httpReq.WithContext(ctx) + httpReq = httpReq.WithContext(ctx) // go1.13 can use NewRequestWithContext // set the headers we signed and the length httpReq.Header = headers diff --git a/backend/webdav/odrvcookie/fetch.go b/backend/webdav/odrvcookie/fetch.go index 4f276b97f..960eb3b52 100644 --- a/backend/webdav/odrvcookie/fetch.go +++ b/backend/webdav/odrvcookie/fetch.go @@ -3,6 +3,7 @@ package odrvcookie import ( "bytes" + "context" "encoding/xml" "html/template" "net/http" @@ -91,8 +92,8 @@ func New(pUser, pPass, pEndpoint string) CookieAuth { // Cookies creates a CookieResponse. It fetches the auth token and then // retrieves the Cookies -func (ca *CookieAuth) Cookies() (*CookieResponse, error) { - tokenResp, err := ca.getSPToken() +func (ca *CookieAuth) Cookies(ctx context.Context) (*CookieResponse, error) { + tokenResp, err := ca.getSPToken(ctx) if err != nil { return nil, err } @@ -140,7 +141,7 @@ func (ca *CookieAuth) getSPCookie(conf *SuccessResponse) (*CookieResponse, error return &cookieResponse, nil } -func (ca *CookieAuth) getSPToken() (conf *SuccessResponse, err error) { +func (ca *CookieAuth) getSPToken(ctx context.Context) (conf *SuccessResponse, err error) { reqData := map[string]interface{}{ "Username": ca.user, "Password": ca.pass, @@ -160,6 +161,7 @@ func (ca *CookieAuth) getSPToken() (conf *SuccessResponse, err error) { if err != nil { return nil, err } + req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext client := fshttp.NewClient(fs.Config) resp, err := client.Do(req) diff --git a/backend/webdav/webdav.go b/backend/webdav/webdav.go index cd2eef089..6d6d20b49 100644 --- a/backend/webdav/webdav.go +++ b/backend/webdav/webdav.go @@ -353,7 +353,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { } } f.srv.SetErrorHandler(errorHandler) - err = f.setQuirks(opt.Vendor) + err = f.setQuirks(ctx, opt.Vendor) if err != nil { return nil, err } @@ -424,7 +424,7 @@ func (f *Fs) fetchAndSetBearerToken() error { } // setQuirks adjusts the Fs for the vendor passed in -func (f *Fs) setQuirks(vendor string) error { +func (f *Fs) setQuirks(ctx context.Context, vendor string) error { switch vendor { case "owncloud": f.canStream = true @@ -440,13 +440,13 @@ func (f *Fs) setQuirks(vendor string) error { // They have to be set instead of BasicAuth f.srv.RemoveHeader("Authorization") // We don't need this Header if using cookies spCk := odrvcookie.New(f.opt.User, f.opt.Pass, f.endpointURL) - spCookies, err := spCk.Cookies() + spCookies, err := spCk.Cookies(ctx) if err != nil { return err } odrvcookie.NewRenew(12*time.Hour, func() { - spCookies, err := spCk.Cookies() + spCookies, err := spCk.Cookies(ctx) if err != nil { fs.Errorf("could not renew cookies: %s", err.Error()) return diff --git a/cmd/rc/rc.go b/cmd/rc/rc.go index b60875cb7..b1661ca64 100644 --- a/cmd/rc/rc.go +++ b/cmd/rc/rc.go @@ -71,11 +71,12 @@ Use "rclone rc" to see a list of all possible commands.`, Run: func(command *cobra.Command, args []string) { cmd.CheckArgs(0, 1e9, command, args) cmd.Run(false, false, command, func() error { + ctx := context.Background() parseFlags() if len(args) == 0 { - return list() + return list(ctx) } - return run(args) + return run(ctx, args) }) }, } @@ -110,7 +111,7 @@ func setAlternateFlag(flagName string, output *string) { // do a call from (path, in) to (out, err). // // if err is set, out may be a valid error return or it may be nil -func doCall(path string, in rc.Params) (out rc.Params, err error) { +func doCall(ctx context.Context, path string, in rc.Params) (out rc.Params, err error) { // If loopback set, short circuit HTTP request if loopback { call := rc.Calls.Get(path) @@ -141,6 +142,7 @@ func doCall(path string, in rc.Params) (out rc.Params, err error) { if err != nil { return nil, errors.Wrap(err, "failed to make request") } + req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext req.Header.Set("Content-Type", "application/json") if authUser != "" || authPass != "" { @@ -182,7 +184,7 @@ func doCall(path string, in rc.Params) (out rc.Params, err error) { } // Run the remote control command passed in -func run(args []string) (err error) { +func run(ctx context.Context, args []string) (err error) { path := strings.Trim(args[0], "/") // parse input @@ -208,7 +210,7 @@ func run(args []string) (err error) { } // Do the call - out, callErr := doCall(path, in) + out, callErr := doCall(ctx, path, in) // Write the JSON blob to stdout if required if out != nil && !noOutput { @@ -222,8 +224,8 @@ func run(args []string) (err error) { } // List the available commands to stdout -func list() error { - list, err := doCall("rc/list", nil) +func list(ctx context.Context) error { + list, err := doCall(ctx, "rc/list", nil) if err != nil { return errors.Wrap(err, "failed to list") } diff --git a/lib/rest/rest.go b/lib/rest/rest.go index 3626060cf..1a746f090 100644 --- a/lib/rest/rest.go +++ b/lib/rest/rest.go @@ -212,10 +212,11 @@ func (api *Client) Call(ctx context.Context, opts *Opts) (resp *http.Response, e if opts.ContentLength != nil && *opts.ContentLength == 0 { body = nil } - req, err := http.NewRequestWithContext(ctx, opts.Method, url, body) + req, err := http.NewRequest(opts.Method, url, body) if err != nil { return } + req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext headers := make(map[string]string) // Set default headers for k, v := range api.headers {