community
/
rclone
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix excessive retries missing --max-duration timeout - fixes #4504 

This change checks the context whenever rclone might retry, and
doesn't retry if the current context has an error.

This fixes the pathological behaviour of `--max-duration` refusing to
exit because all the context deadline exceeded errors were being
retried.

This unfortunately meant changing the shouldRetry logic in every
backend and doing a lot of context propagation.

See: https://forum.rclone.org/t/add-flag-to-exit-immediately-when-max-duration-reached/22723
This commit is contained in:
Nick Craig-Wood 2021-03-11 14:44:01 +00:00
parent 32925dae1f
commit 4013bc4a4c
31 changed files with 474 additions and 360 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

85
backend/amazonclouddrive/amazonclouddrive.go
View File

@ -205,7 +205,10 @@ var retryErrorCodes = []int{
// shouldRetry returns a boolean as to whether this resp and err // shouldRetry returns a boolean as to whether this resp and err
// deserve to be retried. It returns the err as a convenience // deserve to be retried. It returns the err as a convenience
func (f *Fs) shouldRetry(resp *http.Response, err error) (bool, error) { func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
if fserrors.ContextError(ctx, &err) {
return false, err
}
if resp != nil { if resp != nil {
if resp.StatusCode == 401 { if resp.StatusCode == 401 {
f.tokenRenewer.Invalidate() f.tokenRenewer.Invalidate()
@ -280,7 +283,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
// Renew the token in the background // Renew the token in the background
f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error { f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error {
_, err := f.getRootInfo() _, err := f.getRootInfo(ctx)
return err return err
}) })
@ -288,14 +291,14 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
_, resp, err = f.c.Account.GetEndpoints() _, resp, err = f.c.Account.GetEndpoints()
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, errors.Wrap(err, "failed to get endpoints") return nil, errors.Wrap(err, "failed to get endpoints")
} }
// Get rootID // Get rootID
rootInfo, err := f.getRootInfo() rootInfo, err := f.getRootInfo(ctx)
if err != nil || rootInfo.Id == nil { if err != nil || rootInfo.Id == nil {
return nil, errors.Wrap(err, "failed to get root") return nil, errors.Wrap(err, "failed to get root")
} }
@ -337,11 +340,11 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
} }
// getRootInfo gets the root folder info // getRootInfo gets the root folder info
func (f *Fs) getRootInfo() (rootInfo *acd.Folder, err error) { func (f *Fs) getRootInfo(ctx context.Context) (rootInfo *acd.Folder, err error) {
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
rootInfo, resp, err = f.c.Nodes.GetRoot() rootInfo, resp, err = f.c.Nodes.GetRoot()
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
return rootInfo, err return rootInfo, err
} }
@ -380,7 +383,7 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin
var subFolder *acd.Folder var subFolder *acd.Folder
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
subFolder, resp, err = folder.GetFolder(f.opt.Enc.FromStandardName(leaf)) subFolder, resp, err = folder.GetFolder(f.opt.Enc.FromStandardName(leaf))
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
if err == acd.ErrorNodeNotFound { if err == acd.ErrorNodeNotFound {
@ -407,7 +410,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
var info *acd.Folder var info *acd.Folder
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
info, resp, err = folder.CreateFolder(f.opt.Enc.FromStandardName(leaf)) info, resp, err = folder.CreateFolder(f.opt.Enc.FromStandardName(leaf))
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
//fmt.Printf("...Error %v\n", err) //fmt.Printf("...Error %v\n", err)
@ -428,7 +431,7 @@ type listAllFn func(*acd.Node) bool
// Lists the directory required calling the user function on each item found // Lists the directory required calling the user function on each item found
// //
// If the user fn ever returns true then it early exits with found = true // If the user fn ever returns true then it early exits with found = true
func (f *Fs) listAll(dirID string, title string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) { func (f *Fs) listAll(ctx context.Context, dirID string, title string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) {
query := "parents:" + dirID query := "parents:" + dirID
if directoriesOnly { if directoriesOnly {
query += " AND kind:" + folderKind query += " AND kind:" + folderKind
@ -449,7 +452,7 @@ func (f *Fs) listAll(dirID string, title string, directoriesOnly bool, filesOnly
var resp *http.Response var resp *http.Response
err = f.pacer.CallNoRetry(func() (bool, error) { err = f.pacer.CallNoRetry(func() (bool, error) {
nodes, resp, err = f.c.Nodes.GetNodes(&opts) nodes, resp, err = f.c.Nodes.GetNodes(&opts)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return false, err return false, err
@ -508,7 +511,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
var iErr error var iErr error
for tries := 1; tries <= maxTries; tries++ { for tries := 1; tries <= maxTries; tries++ {
entries = nil entries = nil
_, err = f.listAll(directoryID, "", false, false, func(node *acd.Node) bool { _, err = f.listAll(ctx, directoryID, "", false, false, func(node *acd.Node) bool {
remote := path.Join(dir, *node.Name) remote := path.Join(dir, *node.Name)
switch *node.Kind { switch *node.Kind {
case folderKind: case folderKind:
@ -667,7 +670,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
if ok { if ok {
return false, nil return false, nil
} }
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@ -708,7 +711,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
if err != nil { if err != nil {
return nil, err return nil, err
} }
err = f.moveNode(srcObj.remote, dstLeaf, dstDirectoryID, srcObj.info, srcLeaf, srcDirectoryID, false) err = f.moveNode(ctx, srcObj.remote, dstLeaf, dstDirectoryID, srcObj.info, srcLeaf, srcDirectoryID, false)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -803,7 +806,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
var jsonStr string var jsonStr string
err = srcFs.pacer.Call(func() (bool, error) { err = srcFs.pacer.Call(func() (bool, error) {
jsonStr, err = srcInfo.GetMetadata() jsonStr, err = srcInfo.GetMetadata()
return srcFs.shouldRetry(nil, err) return srcFs.shouldRetry(ctx, nil, err)
}) })
if err != nil { if err != nil {
fs.Debugf(src, "DirMove error: error reading src metadata: %v", err) fs.Debugf(src, "DirMove error: error reading src metadata: %v", err)
@ -815,7 +818,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
return err return err
} }
err = f.moveNode(srcPath, dstLeaf, dstDirectoryID, srcInfo, srcLeaf, srcDirectoryID, true) err = f.moveNode(ctx, srcPath, dstLeaf, dstDirectoryID, srcInfo, srcLeaf, srcDirectoryID, true)
if err != nil { if err != nil {
return err return err
} }
@ -840,7 +843,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
if check { if check {
// check directory is empty // check directory is empty
empty := true empty := true
_, err = f.listAll(rootID, "", false, false, func(node *acd.Node) bool { _, err = f.listAll(ctx, rootID, "", false, false, func(node *acd.Node) bool {
switch *node.Kind { switch *node.Kind {
case folderKind: case folderKind:
empty = false empty = false
@ -865,7 +868,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = node.Trash() resp, err = node.Trash()
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return err return err
@ -987,7 +990,7 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
var info *acd.File var info *acd.File
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
info, resp, err = folder.GetFile(o.fs.opt.Enc.FromStandardName(leaf)) info, resp, err = folder.GetFile(o.fs.opt.Enc.FromStandardName(leaf))
return o.fs.shouldRetry(resp, err) return o.fs.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
if err == acd.ErrorNodeNotFound { if err == acd.ErrorNodeNotFound {
@ -1044,7 +1047,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
} else { } else {
in, resp, err = file.OpenTempURLHeaders(o.fs.noAuthClient, headers) in, resp, err = file.OpenTempURLHeaders(o.fs.noAuthClient, headers)
} }
return o.fs.shouldRetry(resp, err) return o.fs.shouldRetry(ctx, resp, err)
}) })
return in, err return in, err
} }
@ -1067,7 +1070,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
if ok { if ok {
return false, nil return false, nil
} }
return o.fs.shouldRetry(resp, err) return o.fs.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return err return err
@ -1077,70 +1080,70 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
} }
// Remove a node // Remove a node
func (f *Fs) removeNode(info *acd.Node) error { func (f *Fs) removeNode(ctx context.Context, info *acd.Node) error {
var resp *http.Response var resp *http.Response
var err error var err error
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = info.Trash() resp, err = info.Trash()
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
return err return err
} }
// Remove an object // Remove an object
func (o *Object) Remove(ctx context.Context) error { func (o *Object) Remove(ctx context.Context) error {
return o.fs.removeNode(o.info) return o.fs.removeNode(ctx, o.info)
} }
// Restore a node // Restore a node
func (f *Fs) restoreNode(info *acd.Node) (newInfo *acd.Node, err error) { func (f *Fs) restoreNode(ctx context.Context, info *acd.Node) (newInfo *acd.Node, err error) {
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
newInfo, resp, err = info.Restore() newInfo, resp, err = info.Restore()
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
return newInfo, err return newInfo, err
} }
// Changes name of given node // Changes name of given node
func (f *Fs) renameNode(info *acd.Node, newName string) (newInfo *acd.Node, err error) { func (f *Fs) renameNode(ctx context.Context, info *acd.Node, newName string) (newInfo *acd.Node, err error) {
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
newInfo, resp, err = info.Rename(f.opt.Enc.FromStandardName(newName)) newInfo, resp, err = info.Rename(f.opt.Enc.FromStandardName(newName))
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
return newInfo, err return newInfo, err
} }
// Replaces one parent with another, effectively moving the file. Leaves other // Replaces one parent with another, effectively moving the file. Leaves other
// parents untouched. ReplaceParent cannot be used when the file is trashed. // parents untouched. ReplaceParent cannot be used when the file is trashed.
func (f *Fs) replaceParent(info *acd.Node, oldParentID string, newParentID string) error { func (f *Fs) replaceParent(ctx context.Context, info *acd.Node, oldParentID string, newParentID string) error {
return f.pacer.Call(func() (bool, error) { return f.pacer.Call(func() (bool, error) {
resp, err := info.ReplaceParent(oldParentID, newParentID) resp, err := info.ReplaceParent(oldParentID, newParentID)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
} }
// Adds one additional parent to object. // Adds one additional parent to object.
func (f *Fs) addParent(info *acd.Node, newParentID string) error { func (f *Fs) addParent(ctx context.Context, info *acd.Node, newParentID string) error {
return f.pacer.Call(func() (bool, error) { return f.pacer.Call(func() (bool, error) {
resp, err := info.AddParent(newParentID) resp, err := info.AddParent(newParentID)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
} }
// Remove given parent from object, leaving the other possible // Remove given parent from object, leaving the other possible
// parents untouched. Object can end up having no parents. // parents untouched. Object can end up having no parents.
func (f *Fs) removeParent(info *acd.Node, parentID string) error { func (f *Fs) removeParent(ctx context.Context, info *acd.Node, parentID string) error {
return f.pacer.Call(func() (bool, error) { return f.pacer.Call(func() (bool, error) {
resp, err := info.RemoveParent(parentID) resp, err := info.RemoveParent(parentID)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
} }
// moveNode moves the node given from the srcLeaf,srcDirectoryID to // moveNode moves the node given from the srcLeaf,srcDirectoryID to
// the dstLeaf,dstDirectoryID // the dstLeaf,dstDirectoryID
func (f *Fs) moveNode(name, dstLeaf, dstDirectoryID string, srcInfo *acd.Node, srcLeaf, srcDirectoryID string, useDirErrorMsgs bool) (err error) { func (f *Fs) moveNode(ctx context.Context, name, dstLeaf, dstDirectoryID string, srcInfo *acd.Node, srcLeaf, srcDirectoryID string, useDirErrorMsgs bool) (err error) {
// fs.Debugf(name, "moveNode dst(%q,%s) <- src(%q,%s)", dstLeaf, dstDirectoryID, srcLeaf, srcDirectoryID) // fs.Debugf(name, "moveNode dst(%q,%s) <- src(%q,%s)", dstLeaf, dstDirectoryID, srcLeaf, srcDirectoryID)
cantMove := fs.ErrorCantMove cantMove := fs.ErrorCantMove
if useDirErrorMsgs { if useDirErrorMsgs {
@ -1154,7 +1157,7 @@ func (f *Fs) moveNode(name, dstLeaf, dstDirectoryID string, srcInfo *acd.Node, s
if srcLeaf != dstLeaf { if srcLeaf != dstLeaf {
// fs.Debugf(name, "renaming") // fs.Debugf(name, "renaming")
_, err = f.renameNode(srcInfo, dstLeaf) _, err = f.renameNode(ctx, srcInfo, dstLeaf)
if err != nil { if err != nil {
fs.Debugf(name, "Move: quick path rename failed: %v", err) fs.Debugf(name, "Move: quick path rename failed: %v", err)
goto OnConflict goto OnConflict
@ -1162,7 +1165,7 @@ func (f *Fs) moveNode(name, dstLeaf, dstDirectoryID string, srcInfo *acd.Node, s
} }
if srcDirectoryID != dstDirectoryID { if srcDirectoryID != dstDirectoryID {
// fs.Debugf(name, "trying parent replace: %s -> %s", oldParentID, newParentID) // fs.Debugf(name, "trying parent replace: %s -> %s", oldParentID, newParentID)
err = f.replaceParent(srcInfo, srcDirectoryID, dstDirectoryID) err = f.replaceParent(ctx, srcInfo, srcDirectoryID, dstDirectoryID)
if err != nil { if err != nil {
fs.Debugf(name, "Move: quick path parent replace failed: %v", err) fs.Debugf(name, "Move: quick path parent replace failed: %v", err)
return err return err
@ -1175,13 +1178,13 @@ OnConflict:
fs.Debugf(name, "Could not directly rename file, presumably because there was a file with the same name already. Instead, the file will now be trashed where such operations do not cause errors. It will be restored to the correct parent after. If any of the subsequent calls fails, the rename/move will be in an invalid state.") fs.Debugf(name, "Could not directly rename file, presumably because there was a file with the same name already. Instead, the file will now be trashed where such operations do not cause errors. It will be restored to the correct parent after. If any of the subsequent calls fails, the rename/move will be in an invalid state.")
// fs.Debugf(name, "Trashing file") // fs.Debugf(name, "Trashing file")
err = f.removeNode(srcInfo) err = f.removeNode(ctx, srcInfo)
if err != nil { if err != nil {
fs.Debugf(name, "Move: remove node failed: %v", err) fs.Debugf(name, "Move: remove node failed: %v", err)
return err return err
} }
// fs.Debugf(name, "Renaming file") // fs.Debugf(name, "Renaming file")
_, err = f.renameNode(srcInfo, dstLeaf) _, err = f.renameNode(ctx, srcInfo, dstLeaf)
if err != nil { if err != nil {
fs.Debugf(name, "Move: rename node failed: %v", err) fs.Debugf(name, "Move: rename node failed: %v", err)
return err return err
@ -1189,19 +1192,19 @@ OnConflict:
// note: replacing parent is forbidden by API, modifying them individually is // note: replacing parent is forbidden by API, modifying them individually is
// okay though // okay though
// fs.Debugf(name, "Adding target parent") // fs.Debugf(name, "Adding target parent")
err = f.addParent(srcInfo, dstDirectoryID) err = f.addParent(ctx, srcInfo, dstDirectoryID)
if err != nil { if err != nil {
fs.Debugf(name, "Move: addParent failed: %v", err) fs.Debugf(name, "Move: addParent failed: %v", err)
return err return err
} }
// fs.Debugf(name, "removing original parent") // fs.Debugf(name, "removing original parent")
err = f.removeParent(srcInfo, srcDirectoryID) err = f.removeParent(ctx, srcInfo, srcDirectoryID)
if err != nil { if err != nil {
fs.Debugf(name, "Move: removeParent failed: %v", err) fs.Debugf(name, "Move: removeParent failed: %v", err)
return err return err
} }
// fs.Debugf(name, "Restoring") // fs.Debugf(name, "Restoring")
_, err = f.restoreNode(srcInfo) _, err = f.restoreNode(ctx, srcInfo)
if err != nil { if err != nil {
fs.Debugf(name, "Move: restoreNode node failed: %v", err) fs.Debugf(name, "Move: restoreNode node failed: %v", err)
return err return err

33
backend/azureblob/azureblob.go
View File

@ -347,7 +347,10 @@ var retryErrorCodes = []int{
// shouldRetry returns a boolean as to whether this resp and err // shouldRetry returns a boolean as to whether this resp and err
// deserve to be retried. It returns the err as a convenience // deserve to be retried. It returns the err as a convenience
func (f *Fs) shouldRetry(err error) (bool, error) { func (f *Fs) shouldRetry(ctx context.Context, err error) (bool, error) {
if fserrors.ContextError(ctx, &err) {
return false, err
}
// FIXME interpret special errors - more to do here // FIXME interpret special errors - more to do here
if storageErr, ok := err.(azblob.StorageError); ok { if storageErr, ok := err.(azblob.StorageError); ok {
switch storageErr.ServiceCode() { switch storageErr.ServiceCode() {
@ -578,7 +581,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
// Retry as specified by the documentation: // Retry as specified by the documentation:
// https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-to-use-vm-token#retry-guidance // https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-to-use-vm-token#retry-guidance
token, err = GetMSIToken(ctx, userMSI) token, err = GetMSIToken(ctx, userMSI)
return f.shouldRetry(err) return f.shouldRetry(ctx, err)
}) })
if err != nil { if err != nil {
@ -594,7 +597,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
var refreshedToken adal.Token var refreshedToken adal.Token
err := f.imdsPacer.Call(func() (bool, error) { err := f.imdsPacer.Call(func() (bool, error) {
refreshedToken, err = GetMSIToken(ctx, userMSI) refreshedToken, err = GetMSIToken(ctx, userMSI)
return f.shouldRetry(err) return f.shouldRetry(ctx, err)
}) })
if err != nil { if err != nil {
// Failed to refresh. // Failed to refresh.
@ -803,7 +806,7 @@ func (f *Fs) list(ctx context.Context, container, directory, prefix string, addC
err := f.pacer.Call(func() (bool, error) { err := f.pacer.Call(func() (bool, error) {
var err error var err error
response, err = f.cntURL(container).ListBlobsHierarchySegment(ctx, marker, delimiter, options) response, err = f.cntURL(container).ListBlobsHierarchySegment(ctx, marker, delimiter, options)
return f.shouldRetry(err) return f.shouldRetry(ctx, err)
}) })
if err != nil { if err != nil {
@ -1029,7 +1032,7 @@ func (f *Fs) listContainersToFn(fn listContainerFn) error {
err := f.pacer.Call(func() (bool, error) { err := f.pacer.Call(func() (bool, error) {
var err error var err error
response, err = f.svcURL.ListContainersSegment(ctx, marker, params) response, err = f.svcURL.ListContainersSegment(ctx, marker, params)
return f.shouldRetry(err) return f.shouldRetry(ctx, err)
}) })
if err != nil { if err != nil {
return err return err
@ -1098,7 +1101,7 @@ func (f *Fs) makeContainer(ctx context.Context, container string) error {
} }
} }
} }
return f.shouldRetry(err) return f.shouldRetry(ctx, err)
}) })
}, nil) }, nil)
} }
@ -1136,10 +1139,10 @@ func (f *Fs) deleteContainer(ctx context.Context, container string) error {
return false, fs.ErrorDirNotFound return false, fs.ErrorDirNotFound
} }
return f.shouldRetry(err) return f.shouldRetry(ctx, err)
} }
return f.shouldRetry(err) return f.shouldRetry(ctx, err)
}) })
}) })
} }
@ -1212,7 +1215,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
startCopy, err = dstBlobURL.StartCopyFromURL(ctx, *source, nil, azblob.ModifiedAccessConditions{}, options, azblob.AccessTierType(f.opt.AccessTier), nil) startCopy, err = dstBlobURL.StartCopyFromURL(ctx, *source, nil, azblob.ModifiedAccessConditions{}, options, azblob.AccessTierType(f.opt.AccessTier), nil)
return f.shouldRetry(err) return f.shouldRetry(ctx, err)
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@ -1373,7 +1376,7 @@ func (o *Object) readMetaData() (err error) {
var blobProperties *azblob.BlobGetPropertiesResponse var blobProperties *azblob.BlobGetPropertiesResponse
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
blobProperties, err = blob.GetProperties(ctx, options, azblob.ClientProvidedKeyOptions{}) blobProperties, err = blob.GetProperties(ctx, options, azblob.ClientProvidedKeyOptions{})
return o.fs.shouldRetry(err) return o.fs.shouldRetry(ctx, err)
}) })
if err != nil { if err != nil {
// On directories - GetProperties does not work and current SDK does not populate service code correctly hence check regular http response as well // On directories - GetProperties does not work and current SDK does not populate service code correctly hence check regular http response as well
@ -1408,7 +1411,7 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
blob := o.getBlobReference() blob := o.getBlobReference()
err := o.fs.pacer.Call(func() (bool, error) { err := o.fs.pacer.Call(func() (bool, error) {
_, err := blob.SetMetadata(ctx, o.meta, azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{}) _, err := blob.SetMetadata(ctx, o.meta, azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
return o.fs.shouldRetry(err) return o.fs.shouldRetry(ctx, err)
}) })
if err != nil { if err != nil {
return err return err
@ -1451,7 +1454,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
var downloadResponse *azblob.DownloadResponse var downloadResponse *azblob.DownloadResponse
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
downloadResponse, err = blob.Download(ctx, offset, count, ac, false, azblob.ClientProvidedKeyOptions{}) downloadResponse, err = blob.Download(ctx, offset, count, ac, false, azblob.ClientProvidedKeyOptions{})
return o.fs.shouldRetry(err) return o.fs.shouldRetry(ctx, err)
}) })
if err != nil { if err != nil {
return nil, errors.Wrap(err, "failed to open for download") return nil, errors.Wrap(err, "failed to open for download")
@ -1592,7 +1595,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
// Stream contents of the reader object to the given blob URL // Stream contents of the reader object to the given blob URL
blockBlobURL := blob.ToBlockBlobURL() blockBlobURL := blob.ToBlockBlobURL()
_, err = azblob.UploadStreamToBlockBlob(ctx, in, blockBlobURL, putBlobOptions) _, err = azblob.UploadStreamToBlockBlob(ctx, in, blockBlobURL, putBlobOptions)
return o.fs.shouldRetry(err) return o.fs.shouldRetry(ctx, err)
}) })
if err != nil { if err != nil {
return err return err
@ -1620,7 +1623,7 @@ func (o *Object) Remove(ctx context.Context) error {
ac := azblob.BlobAccessConditions{} ac := azblob.BlobAccessConditions{}
return o.fs.pacer.Call(func() (bool, error) { return o.fs.pacer.Call(func() (bool, error) {
_, err := blob.Delete(ctx, snapShotOptions, ac) _, err := blob.Delete(ctx, snapShotOptions, ac)
return o.fs.shouldRetry(err) return o.fs.shouldRetry(ctx, err)
}) })
} }
@ -1649,7 +1652,7 @@ func (o *Object) SetTier(tier string) error {
ctx := context.Background() ctx := context.Background()
err := o.fs.pacer.Call(func() (bool, error) { err := o.fs.pacer.Call(func() (bool, error) {
_, err := blob.SetTier(ctx, desiredAccessTier, azblob.LeaseAccessConditions{}) _, err := blob.SetTier(ctx, desiredAccessTier, azblob.LeaseAccessConditions{})
return o.fs.shouldRetry(err) return o.fs.shouldRetry(ctx, err)
}) })
if err != nil { if err != nil {

9
backend/b2/b2.go
View File

@ -305,7 +305,10 @@ var retryErrorCodes = []int{
// shouldRetryNoAuth returns a boolean as to whether this resp and err // shouldRetryNoAuth returns a boolean as to whether this resp and err
// deserve to be retried. It returns the err as a convenience // deserve to be retried. It returns the err as a convenience
func (f *Fs) shouldRetryNoReauth(resp *http.Response, err error) (bool, error) { func (f *Fs) shouldRetryNoReauth(ctx context.Context, resp *http.Response, err error) (bool, error) {
if fserrors.ContextError(ctx, &err) {
return false, err
}
// For 429 or 503 errors look at the Retry-After: header and // For 429 or 503 errors look at the Retry-After: header and
// set the retry appropriately, starting with a minimum of 1 // set the retry appropriately, starting with a minimum of 1
// second if it isn't set. // second if it isn't set.
@ -336,7 +339,7 @@ func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (b
} }
return true, err return true, err
} }
return f.shouldRetryNoReauth(resp, err) return f.shouldRetryNoReauth(ctx, resp, err)
} }
// errorHandler parses a non 2xx error response into an error // errorHandler parses a non 2xx error response into an error
@ -504,7 +507,7 @@ func (f *Fs) authorizeAccount(ctx context.Context) error {
} }
err := f.pacer.Call(func() (bool, error) { err := f.pacer.Call(func() (bool, error) {
resp, err := f.srv.CallJSON(ctx, &opts, nil, &f.info) resp, err := f.srv.CallJSON(ctx, &opts, nil, &f.info)
return f.shouldRetryNoReauth(resp, err) return f.shouldRetryNoReauth(ctx, resp, err)
}) })
if err != nil { if err != nil {
return errors.Wrap(err, "failed to authenticate") return errors.Wrap(err, "failed to authenticate")

31
backend/box/box.go
View File

@ -317,7 +317,10 @@ var retryErrorCodes = []int{
// shouldRetry returns a boolean as to whether this resp and err // shouldRetry returns a boolean as to whether this resp and err
// deserve to be retried. It returns the err as a convenience // deserve to be retried. It returns the err as a convenience
func shouldRetry(resp *http.Response, err error) (bool, error) { func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
if fserrors.ContextError(ctx, &err) {
return false, err
}
authRetry := false authRetry := false
if resp != nil && resp.StatusCode == 401 && len(resp.Header["Www-Authenticate"]) == 1 && strings.Index(resp.Header["Www-Authenticate"][0], "expired_token") >= 0 { if resp != nil && resp.StatusCode == 401 && len(resp.Header["Www-Authenticate"]) == 1 && strings.Index(resp.Header["Www-Authenticate"][0], "expired_token") >= 0 {
@ -548,7 +551,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
} }
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, &mkdir, &info) resp, err = f.srv.CallJSON(ctx, &opts, &mkdir, &info)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
//fmt.Printf("...Error %v\n", err) //fmt.Printf("...Error %v\n", err)
@ -585,7 +588,7 @@ OUTER:
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return found, errors.Wrap(err, "couldn't list files") return found, errors.Wrap(err, "couldn't list files")
@ -740,7 +743,7 @@ func (f *Fs) deleteObject(ctx context.Context, id string) error {
} }
return f.pacer.Call(func() (bool, error) { return f.pacer.Call(func() (bool, error) {
resp, err := f.srv.Call(ctx, &opts) resp, err := f.srv.Call(ctx, &opts)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
} }
@ -767,7 +770,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.Call(ctx, &opts) resp, err = f.srv.Call(ctx, &opts)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return errors.Wrap(err, "rmdir failed") return errors.Wrap(err, "rmdir failed")
@ -839,7 +842,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
var info *api.Item var info *api.Item
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, &copyFile, &info) resp, err = f.srv.CallJSON(ctx, &opts, &copyFile, &info)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@ -877,7 +880,7 @@ func (f *Fs) move(ctx context.Context, endpoint, id, leaf, directoryID string) (
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, &move, &info) resp, err = f.srv.CallJSON(ctx, &opts, &move, &info)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@ -895,7 +898,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &user) resp, err = f.srv.CallJSON(ctx, &opts, nil, &user)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, errors.Wrap(err, "failed to read user info") return nil, errors.Wrap(err, "failed to read user info")
@ -1008,7 +1011,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, &shareLink, &info) resp, err = f.srv.CallJSON(ctx, &opts, &shareLink, &info)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
return info.SharedLink.URL, err return info.SharedLink.URL, err
} }
@ -1026,7 +1029,7 @@ func (f *Fs) deletePermanently(ctx context.Context, itemType, id string) error {
} }
return f.pacer.Call(func() (bool, error) { return f.pacer.Call(func() (bool, error) {
resp, err := f.srv.Call(ctx, &opts) resp, err := f.srv.Call(ctx, &opts)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
} }
@ -1048,7 +1051,7 @@ func (f *Fs) CleanUp(ctx context.Context) (err error) {
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return errors.Wrap(err, "couldn't list trash") return errors.Wrap(err, "couldn't list trash")
@ -1182,7 +1185,7 @@ func (o *Object) setModTime(ctx context.Context, modTime time.Time) (*api.Item,
var info *api.Item var info *api.Item
err := o.fs.pacer.Call(func() (bool, error) { err := o.fs.pacer.Call(func() (bool, error) {
resp, err := o.fs.srv.CallJSON(ctx, &opts, &update, &info) resp, err := o.fs.srv.CallJSON(ctx, &opts, &update, &info)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
return info, err return info, err
} }
@ -1215,7 +1218,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
} }
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.Call(ctx, &opts) resp, err = o.fs.srv.Call(ctx, &opts)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@ -1255,7 +1258,7 @@ func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID str
} }
err = o.fs.pacer.CallNoRetry(func() (bool, error) { err = o.fs.pacer.CallNoRetry(func() (bool, error) {
resp, err = o.fs.srv.CallJSON(ctx, &opts, &upload, &result) resp, err = o.fs.srv.CallJSON(ctx, &opts, &upload, &result)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return err return err

10
backend/box/upload.go
View File

@ -44,7 +44,7 @@ func (o *Object) createUploadSession(ctx context.Context, leaf, directoryID stri
var resp *http.Response var resp *http.Response
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.CallJSON(ctx, &opts, &request, &response) resp, err = o.fs.srv.CallJSON(ctx, &opts, &request, &response)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
return return
} }
@ -74,7 +74,7 @@ func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, total
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
opts.Body = wrap(bytes.NewReader(chunk)) opts.Body = wrap(bytes.NewReader(chunk))
resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &response) resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &response)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@ -109,10 +109,10 @@ outer:
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.CallJSON(ctx, &opts, &request, nil) resp, err = o.fs.srv.CallJSON(ctx, &opts, &request, nil)
if err != nil { if err != nil {
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
} }
body, err = rest.ReadBody(resp) body, err = rest.ReadBody(resp)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
delay := defaultDelay delay := defaultDelay
var why string var why string
@ -167,7 +167,7 @@ func (o *Object) abortUpload(ctx context.Context, SessionID string) (err error)
var resp *http.Response var resp *http.Response
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.Call(ctx, &opts) resp, err = o.fs.srv.Call(ctx, &opts)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
return err return err
} }

135
backend/dropbox/dropbox.go
View File

@ -292,7 +292,10 @@ func (f *Fs) Features() *fs.Features {
// shouldRetry returns a boolean as to whether this err deserves to be // shouldRetry returns a boolean as to whether this err deserves to be
// retried. It returns the err as a convenience // retried. It returns the err as a convenience
func shouldRetry(err error) (bool, error) { func shouldRetry(ctx context.Context, err error) (bool, error) {
if fserrors.ContextError(ctx, &err) {
return false, err
}
if err == nil { if err == nil {
return false, err return false, err
} }
@ -425,7 +428,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
if f.root == "" { if f.root == "" {
return f, nil return f, nil
} }
_, err := f.findSharedFile(f.root) _, err := f.findSharedFile(ctx, f.root)
f.root = "" f.root = ""
if err == nil { if err == nil {
return f, fs.ErrorIsFile return f, fs.ErrorIsFile
@ -445,7 +448,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
} }
// root is not empty so we have find the right shared folder if it exists // root is not empty so we have find the right shared folder if it exists
id, err := f.findSharedFolder(dir) id, err := f.findSharedFolder(ctx, dir)
if err != nil { if err != nil {
// if we didn't find the specified shared folder we have to bail out here // if we didn't find the specified shared folder we have to bail out here
return nil, err return nil, err
@ -453,7 +456,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
// we found the specified shared folder so let's mount it // we found the specified shared folder so let's mount it
// this will add it to the users normal root namespace and allows us // this will add it to the users normal root namespace and allows us
// to actually perform operations on it using the normal api endpoints. // to actually perform operations on it using the normal api endpoints.
err = f.mountSharedFolder(id) err = f.mountSharedFolder(ctx, id)
if err != nil { if err != nil {
switch e := err.(type) { switch e := err.(type) {
case sharing.MountFolderAPIError: case sharing.MountFolderAPIError:
@ -477,7 +480,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
var acc *users.FullAccount var acc *users.FullAccount
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
acc, err = f.users.GetCurrentAccount() acc, err = f.users.GetCurrentAccount()
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
if err != nil { if err != nil {
return nil, errors.Wrap(err, "get current account failed") return nil, errors.Wrap(err, "get current account failed")
@ -495,7 +498,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
f.setRoot(root) f.setRoot(root)
// See if the root is actually an object // See if the root is actually an object
_, err = f.getFileMetadata(f.slashRoot) _, err = f.getFileMetadata(ctx, f.slashRoot)
if err == nil { if err == nil {
newRoot := path.Dir(f.root) newRoot := path.Dir(f.root)
if newRoot == "." { if newRoot == "." {
@ -529,12 +532,12 @@ func (f *Fs) setRoot(root string) {
} }
// getMetadata gets the metadata for a file or directory // getMetadata gets the metadata for a file or directory
func (f *Fs) getMetadata(objPath string) (entry files.IsMetadata, notFound bool, err error) { func (f *Fs) getMetadata(ctx context.Context, objPath string) (entry files.IsMetadata, notFound bool, err error) {
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
entry, err = f.srv.GetMetadata(&files.GetMetadataArg{ entry, err = f.srv.GetMetadata(&files.GetMetadataArg{
Path: f.opt.Enc.FromStandardPath(objPath), Path: f.opt.Enc.FromStandardPath(objPath),
}) })
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
if err != nil { if err != nil {
switch e := err.(type) { switch e := err.(type) {
@ -549,8 +552,8 @@ func (f *Fs) getMetadata(objPath string) (entry files.IsMetadata, notFound bool,
} }
// getFileMetadata gets the metadata for a file // getFileMetadata gets the metadata for a file
func (f *Fs) getFileMetadata(filePath string) (fileInfo *files.FileMetadata, err error) { func (f *Fs) getFileMetadata(ctx context.Context, filePath string) (fileInfo *files.FileMetadata, err error) {
entry, notFound, err := f.getMetadata(filePath) entry, notFound, err := f.getMetadata(ctx, filePath)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -565,8 +568,8 @@ func (f *Fs) getFileMetadata(filePath string) (fileInfo *files.FileMetadata, err
} }
// getDirMetadata gets the metadata for a directory // getDirMetadata gets the metadata for a directory
func (f *Fs) getDirMetadata(dirPath string) (dirInfo *files.FolderMetadata, err error) { func (f *Fs) getDirMetadata(ctx context.Context, dirPath string) (dirInfo *files.FolderMetadata, err error) {
entry, notFound, err := f.getMetadata(dirPath) entry, notFound, err := f.getMetadata(ctx, dirPath)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -583,7 +586,7 @@ func (f *Fs) getDirMetadata(dirPath string) (dirInfo *files.FolderMetadata, err
// Return an Object from a path // Return an Object from a path
// //
// If it can't be found it returns the error fs.ErrorObjectNotFound. // If it can't be found it returns the error fs.ErrorObjectNotFound.
func (f *Fs) newObjectWithInfo(remote string, info *files.FileMetadata) (fs.Object, error) { func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *files.FileMetadata) (fs.Object, error) {
o := &Object{ o := &Object{
fs: f, fs: f,
remote: remote, remote: remote,
@ -592,7 +595,7 @@ func (f *Fs) newObjectWithInfo(remote string, info *files.FileMetadata) (fs.Obje
if info != nil { if info != nil {
err = o.setMetadataFromEntry(info) err = o.setMetadataFromEntry(info)
} else { } else {
err = o.readEntryAndSetMetadata() err = o.readEntryAndSetMetadata(ctx)
} }
if err != nil { if err != nil {
return nil, err return nil, err
@ -604,14 +607,14 @@ func (f *Fs) newObjectWithInfo(remote string, info *files.FileMetadata) (fs.Obje
// it returns the error fs.ErrorObjectNotFound. // it returns the error fs.ErrorObjectNotFound.
func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
if f.opt.SharedFiles { if f.opt.SharedFiles {
return f.findSharedFile(remote) return f.findSharedFile(ctx, remote)
} }
return f.newObjectWithInfo(remote, nil) return f.newObjectWithInfo(ctx, remote, nil)
} }
// listSharedFoldersApi lists all available shared folders mounted and not mounted // listSharedFoldersApi lists all available shared folders mounted and not mounted
// we'll need the id later so we have to return them in original format // we'll need the id later so we have to return them in original format
func (f *Fs) listSharedFolders() (entries fs.DirEntries, err error) { func (f *Fs) listSharedFolders(ctx context.Context) (entries fs.DirEntries, err error) {
started := false started := false
var res *sharing.ListFoldersResult var res *sharing.ListFoldersResult
for { for {
@ -621,7 +624,7 @@ func (f *Fs) listSharedFolders() (entries fs.DirEntries, err error) {
} }
err := f.pacer.Call(func() (bool, error) { err := f.pacer.Call(func() (bool, error) {
res, err = f.sharing.ListFolders(&arg) res, err = f.sharing.ListFolders(&arg)
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@ -633,7 +636,7 @@ func (f *Fs) listSharedFolders() (entries fs.DirEntries, err error) {
} }
err := f.pacer.Call(func() (bool, error) { err := f.pacer.Call(func() (bool, error) {
res, err = f.sharing.ListFoldersContinue(&arg) res, err = f.sharing.ListFoldersContinue(&arg)
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
if err != nil { if err != nil {
return nil, errors.Wrap(err, "list continue") return nil, errors.Wrap(err, "list continue")
@ -658,8 +661,8 @@ func (f *Fs) listSharedFolders() (entries fs.DirEntries, err error) {
// findSharedFolder find the id for a given shared folder name // findSharedFolder find the id for a given shared folder name
// somewhat annoyingly there is no endpoint to query a shared folder by it's name // somewhat annoyingly there is no endpoint to query a shared folder by it's name
// so our only option is to iterate over all shared folders // so our only option is to iterate over all shared folders
func (f *Fs) findSharedFolder(name string) (id string, err error) { func (f *Fs) findSharedFolder(ctx context.Context, name string) (id string, err error) {
entries, err := f.listSharedFolders() entries, err := f.listSharedFolders(ctx)
if err != nil { if err != nil {
return "", err return "", err
} }
@ -672,20 +675,20 @@ func (f *Fs) findSharedFolder(name string) (id string, err error) {
} }
// mountSharedFolder mount a shared folder to the root namespace // mountSharedFolder mount a shared folder to the root namespace
func (f *Fs) mountSharedFolder(id string) error { func (f *Fs) mountSharedFolder(ctx context.Context, id string) error {
arg := sharing.MountFolderArg{ arg := sharing.MountFolderArg{
SharedFolderId: id, SharedFolderId: id,
} }
err := f.pacer.Call(func() (bool, error) { err := f.pacer.Call(func() (bool, error) {
_, err := f.sharing.MountFolder(&arg) _, err := f.sharing.MountFolder(&arg)
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
return err return err
} }
// listReceivedFiles lists shared the user as access to (note this means individual // listReceivedFiles lists shared the user as access to (note this means individual
// files not files contained in shared folders) // files not files contained in shared folders)
func (f *Fs) listReceivedFiles() (entries fs.DirEntries, err error) { func (f *Fs) listReceivedFiles(ctx context.Context) (entries fs.DirEntries, err error) {
started := false started := false
var res *sharing.ListFilesResult var res *sharing.ListFilesResult
for { for {
@ -695,7 +698,7 @@ func (f *Fs) listReceivedFiles() (entries fs.DirEntries, err error) {
} }
err := f.pacer.Call(func() (bool, error) { err := f.pacer.Call(func() (bool, error) {
res, err = f.sharing.ListReceivedFiles(&arg) res, err = f.sharing.ListReceivedFiles(&arg)
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@ -707,7 +710,7 @@ func (f *Fs) listReceivedFiles() (entries fs.DirEntries, err error) {
} }
err := f.pacer.Call(func() (bool, error) { err := f.pacer.Call(func() (bool, error) {
res, err = f.sharing.ListReceivedFilesContinue(&arg) res, err = f.sharing.ListReceivedFilesContinue(&arg)
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
if err != nil { if err != nil {
return nil, errors.Wrap(err, "list continue") return nil, errors.Wrap(err, "list continue")
@ -734,8 +737,8 @@ func (f *Fs) listReceivedFiles() (entries fs.DirEntries, err error) {
return entries, nil return entries, nil
} }
func (f *Fs) findSharedFile(name string) (o *Object, err error) { func (f *Fs) findSharedFile(ctx context.Context, name string) (o *Object, err error) {
files, err := f.listReceivedFiles() files, err := f.listReceivedFiles(ctx)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -758,10 +761,10 @@ func (f *Fs) findSharedFile(name string) (o *Object, err error) {
// found. // found.
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) { func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
if f.opt.SharedFiles { if f.opt.SharedFiles {
return f.listReceivedFiles() return f.listReceivedFiles(ctx)
} }
if f.opt.SharedFolders { if f.opt.SharedFolders {
return f.listSharedFolders() return f.listSharedFolders(ctx)
} }
root := f.slashRoot root := f.slashRoot
@ -782,7 +785,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
} }
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
res, err = f.srv.ListFolder(&arg) res, err = f.srv.ListFolder(&arg)
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
if err != nil { if err != nil {
switch e := err.(type) { switch e := err.(type) {
@ -800,7 +803,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
} }
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
res, err = f.srv.ListFolderContinue(&arg) res, err = f.srv.ListFolderContinue(&arg)
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
if err != nil { if err != nil {
return nil, errors.Wrap(err, "list continue") return nil, errors.Wrap(err, "list continue")
@ -830,7 +833,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
d := fs.NewDir(remote, time.Now()).SetID(folderInfo.Id) d := fs.NewDir(remote, time.Now()).SetID(folderInfo.Id)
entries = append(entries, d) entries = append(entries, d)
} else if fileInfo != nil { } else if fileInfo != nil {
o, err := f.newObjectWithInfo(remote, fileInfo) o, err := f.newObjectWithInfo(ctx, remote, fileInfo)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -879,7 +882,7 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
} }
// check directory doesn't exist // check directory doesn't exist
_, err := f.getDirMetadata(root) _, err := f.getDirMetadata(ctx, root)
if err == nil { if err == nil {
return nil // directory exists already return nil // directory exists already
} else if err != fs.ErrorDirNotFound { } else if err != fs.ErrorDirNotFound {
@ -896,7 +899,7 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
} }
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
_, err = f.srv.CreateFolderV2(&arg2) _, err = f.srv.CreateFolderV2(&arg2)
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
return err return err
} }
@ -913,7 +916,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
if check { if check {
// check directory exists // check directory exists
_, err = f.getDirMetadata(root) _, err = f.getDirMetadata(ctx, root)
if err != nil { if err != nil {
return errors.Wrap(err, "Rmdir") return errors.Wrap(err, "Rmdir")
} }
@ -930,7 +933,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
var res *files.ListFolderResult var res *files.ListFolderResult
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
res, err = f.srv.ListFolder(&arg) res, err = f.srv.ListFolder(&arg)
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
if err != nil { if err != nil {
return errors.Wrap(err, "Rmdir") return errors.Wrap(err, "Rmdir")
@ -943,7 +946,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
// remove it // remove it
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
_, err = f.srv.DeleteV2(&files.DeleteArg{Path: root}) _, err = f.srv.DeleteV2(&files.DeleteArg{Path: root})
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
return err return err
} }
@ -996,7 +999,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
var result *files.RelocationResult var result *files.RelocationResult
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
result, err = f.srv.CopyV2(&arg) result, err = f.srv.CopyV2(&arg)
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
if err != nil { if err != nil {
return nil, errors.Wrap(err, "copy failed") return nil, errors.Wrap(err, "copy failed")
@ -1057,7 +1060,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
var result *files.RelocationResult var result *files.RelocationResult
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
result, err = f.srv.MoveV2(&arg) result, err = f.srv.MoveV2(&arg)
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
if err != nil { if err != nil {
return nil, errors.Wrap(err, "move failed") return nil, errors.Wrap(err, "move failed")
@ -1091,7 +1094,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
var linkRes sharing.IsSharedLinkMetadata var linkRes sharing.IsSharedLinkMetadata
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
linkRes, err = f.sharing.CreateSharedLinkWithSettings(&createArg) linkRes, err = f.sharing.CreateSharedLinkWithSettings(&createArg)
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
if err != nil && strings.Contains(err.Error(), if err != nil && strings.Contains(err.Error(),
@ -1104,7 +1107,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
var listRes *sharing.ListSharedLinksResult var listRes *sharing.ListSharedLinksResult
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
listRes, err = f.sharing.ListSharedLinks(&listArg) listRes, err = f.sharing.ListSharedLinks(&listArg)
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
if err != nil { if err != nil {
return return
@ -1146,7 +1149,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
dstPath := path.Join(f.slashRoot, dstRemote) dstPath := path.Join(f.slashRoot, dstRemote)
// Check if destination exists // Check if destination exists
_, err := f.getDirMetadata(dstPath) _, err := f.getDirMetadata(ctx, dstPath)
if err == nil { if err == nil {
return fs.ErrorDirExists return fs.ErrorDirExists
} else if err != fs.ErrorDirNotFound { } else if err != fs.ErrorDirNotFound {
@ -1165,7 +1168,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
} }
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
_, err = f.srv.MoveV2(&arg) _, err = f.srv.MoveV2(&arg)
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
if err != nil { if err != nil {
return errors.Wrap(err, "MoveDir failed") return errors.Wrap(err, "MoveDir failed")
@ -1179,7 +1182,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
var q *users.SpaceUsage var q *users.SpaceUsage
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
q, err = f.users.GetSpaceUsage() q, err = f.users.GetSpaceUsage()
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
if err != nil { if err != nil {
return nil, errors.Wrap(err, "about failed") return nil, errors.Wrap(err, "about failed")
@ -1210,7 +1213,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryType), pollIntervalChan <-chan time.Duration) { func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryType), pollIntervalChan <-chan time.Duration) {
go func() { go func() {
// get the StartCursor early so all changes from now on get processed // get the StartCursor early so all changes from now on get processed
startCursor, err := f.changeNotifyCursor() startCursor, err := f.changeNotifyCursor(ctx)
if err != nil { if err != nil {
fs.Infof(f, "Failed to get StartCursor: %s", err) fs.Infof(f, "Failed to get StartCursor: %s", err)
} }
@ -1235,7 +1238,7 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT
} }
case <-tickerC: case <-tickerC:
if startCursor == "" { if startCursor == "" {
startCursor, err = f.changeNotifyCursor() startCursor, err = f.changeNotifyCursor(ctx)
if err != nil { if err != nil {
fs.Infof(f, "Failed to get StartCursor: %s", err) fs.Infof(f, "Failed to get StartCursor: %s", err)
continue continue
@ -1251,7 +1254,7 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT
}() }()
} }
func (f *Fs) changeNotifyCursor() (cursor string, err error) { func (f *Fs) changeNotifyCursor(ctx context.Context) (cursor string, err error) {
var startCursor *files.ListFolderGetLatestCursorResult var startCursor *files.ListFolderGetLatestCursorResult
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
@ -1266,7 +1269,7 @@ func (f *Fs) changeNotifyCursor() (cursor string, err error) {
startCursor, err = f.srv.ListFolderGetLatestCursor(&arg) startCursor, err = f.srv.ListFolderGetLatestCursor(&arg)
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
if err != nil { if err != nil {
return return
@ -1296,7 +1299,7 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
} }
res, err = f.svc.ListFolderLongpoll(&args) res, err = f.svc.ListFolderLongpoll(&args)
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
if err != nil { if err != nil {
return return
@ -1319,7 +1322,7 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
} }
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
changeList, err = f.srv.ListFolderContinue(&arg) changeList, err = f.srv.ListFolderContinue(&arg)
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
if err != nil { if err != nil {
return "", errors.Wrap(err, "list continue") return "", errors.Wrap(err, "list continue")
@ -1392,7 +1395,7 @@ func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
if t != DbHashType { if t != DbHashType {
return "", hash.ErrUnsupported return "", hash.ErrUnsupported
} }
err := o.readMetaData() err := o.readMetaData(ctx)
if err != nil { if err != nil {
return "", errors.Wrap(err, "failed to read hash from metadata") return "", errors.Wrap(err, "failed to read hash from metadata")
} }
@ -1416,17 +1419,17 @@ func (o *Object) setMetadataFromEntry(info *files.FileMetadata) error {
} }
// Reads the entry for a file from dropbox // Reads the entry for a file from dropbox
func (o *Object) readEntry() (*files.FileMetadata, error) { func (o *Object) readEntry(ctx context.Context) (*files.FileMetadata, error) {
return o.fs.getFileMetadata(o.remotePath()) return o.fs.getFileMetadata(ctx, o.remotePath())
} }
// Read entry if not set and set metadata from it // Read entry if not set and set metadata from it
func (o *Object) readEntryAndSetMetadata() error { func (o *Object) readEntryAndSetMetadata(ctx context.Context) error {
// Last resort set time from client // Last resort set time from client
if !o.modTime.IsZero() { if !o.modTime.IsZero() {
return nil return nil
} }
entry, err := o.readEntry() entry, err := o.readEntry(ctx)
if err != nil { if err != nil {
return err return err
} }
@ -1439,12 +1442,12 @@ func (o *Object) remotePath() string {
} }
// readMetaData gets the info if it hasn't already been fetched // readMetaData gets the info if it hasn't already been fetched
func (o *Object) readMetaData() (err error) { func (o *Object) readMetaData(ctx context.Context) (err error) {
if !o.modTime.IsZero() { if !o.modTime.IsZero() {
return nil return nil
} }
// Last resort // Last resort
return o.readEntryAndSetMetadata() return o.readEntryAndSetMetadata(ctx)
} }
// ModTime returns the modification time of the object // ModTime returns the modification time of the object
@ -1452,7 +1455,7 @@ func (o *Object) readMetaData() (err error) {
// It attempts to read the objects mtime and if that isn't present the // It attempts to read the objects mtime and if that isn't present the
// LastModified returned in the http headers // LastModified returned in the http headers
func (o *Object) ModTime(ctx context.Context) time.Time { func (o *Object) ModTime(ctx context.Context) time.Time {
err := o.readMetaData() err := o.readMetaData(ctx)
if err != nil { if err != nil {
fs.Debugf(o, "Failed to read metadata: %v", err) fs.Debugf(o, "Failed to read metadata: %v", err)
return time.Now() return time.Now()
@ -1486,7 +1489,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
} }
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
_, in, err = o.fs.sharing.GetSharedLinkFile(&arg) _, in, err = o.fs.sharing.GetSharedLinkFile(&arg)
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@ -1502,7 +1505,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
} }
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
_, in, err = o.fs.srv.Download(&arg) _, in, err = o.fs.srv.Download(&arg)
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
switch e := err.(type) { switch e := err.(type) {
@ -1521,7 +1524,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
// Will work optimally if size is >= uploadChunkSize. If the size is either // Will work optimally if size is >= uploadChunkSize. If the size is either
// unknown (i.e. -1) or smaller than uploadChunkSize, the method incurs an // unknown (i.e. -1) or smaller than uploadChunkSize, the method incurs an
// avoidable request to the Dropbox API that does not carry payload. // avoidable request to the Dropbox API that does not carry payload.
func (o *Object) uploadChunked(in0 io.Reader, commitInfo *files.CommitInfo, size int64) (entry *files.FileMetadata, err error) { func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *files.CommitInfo, size int64) (entry *files.FileMetadata, err error) {
chunkSize := int64(o.fs.opt.ChunkSize) chunkSize := int64(o.fs.opt.ChunkSize)
chunks := 0 chunks := 0
if size != -1 { if size != -1 {
@ -1550,7 +1553,7 @@ func (o *Object) uploadChunked(in0 io.Reader, commitInfo *files.CommitInfo, size
return false, nil return false, nil
} }
res, err = o.fs.srv.UploadSessionStart(&files.UploadSessionStartArg{}, chunk) res, err = o.fs.srv.UploadSessionStart(&files.UploadSessionStartArg{}, chunk)
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@ -1676,11 +1679,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
var err error var err error
var entry *files.FileMetadata var entry *files.FileMetadata
if size > int64(o.fs.opt.ChunkSize) || size == -1 { if size > int64(o.fs.opt.ChunkSize) || size == -1 {
entry, err = o.uploadChunked(in, commitInfo, size) entry, err = o.uploadChunked(ctx, in, commitInfo, size)
} else { } else {
err = o.fs.pacer.CallNoRetry(func() (bool, error) { err = o.fs.pacer.CallNoRetry(func() (bool, error) {
entry, err = o.fs.srv.Upload(commitInfo, in) entry, err = o.fs.srv.Upload(commitInfo, in)
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
} }
if err != nil { if err != nil {
@ -1698,7 +1701,7 @@ func (o *Object) Remove(ctx context.Context) (err error) {
_, err = o.fs.srv.DeleteV2(&files.DeleteArg{ _, err = o.fs.srv.DeleteV2(&files.DeleteArg{
Path: o.fs.opt.Enc.FromStandardPath(o.remotePath()), Path: o.fs.opt.Enc.FromStandardPath(o.remotePath()),
}) })
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
return err return err
} }

31
backend/fichier/api.go
View File

@ -28,7 +28,10 @@ var retryErrorCodes = []int{
// shouldRetry returns a boolean as to whether this resp and err // shouldRetry returns a boolean as to whether this resp and err
// deserve to be retried. It returns the err as a convenience // deserve to be retried. It returns the err as a convenience
func shouldRetry(resp *http.Response, err error) (bool, error) { func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
if fserrors.ContextError(ctx, &err) {
return false, err
}
// Detect this error which the integration tests provoke // Detect this error which the integration tests provoke
// error HTTP error 403 (403 Forbidden) returned body: "{\"message\":\"Flood detected: IP Locked #374\",\"status\":\"KO\"}" // error HTTP error 403 (403 Forbidden) returned body: "{\"message\":\"Flood detected: IP Locked #374\",\"status\":\"KO\"}"
// //
@ -74,7 +77,7 @@ func (f *Fs) readFileInfo(ctx context.Context, url string) (*File, error) {
var file File var file File
err := f.pacer.Call(func() (bool, error) { err := f.pacer.Call(func() (bool, error) {
resp, err := f.rest.CallJSON(ctx, &opts, &request, &file) resp, err := f.rest.CallJSON(ctx, &opts, &request, &file)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, errors.Wrap(err, "couldn't read file info") return nil, errors.Wrap(err, "couldn't read file info")
@ -96,7 +99,7 @@ func (f *Fs) getDownloadToken(ctx context.Context, url string) (*GetTokenRespons
var token GetTokenResponse var token GetTokenResponse
err := f.pacer.Call(func() (bool, error) { err := f.pacer.Call(func() (bool, error) {
resp, err := f.rest.CallJSON(ctx, &opts, &request, &token) resp, err := f.rest.CallJSON(ctx, &opts, &request, &token)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, errors.Wrap(err, "couldn't list files") return nil, errors.Wrap(err, "couldn't list files")
@ -124,7 +127,7 @@ func (f *Fs) listSharedFiles(ctx context.Context, id string) (entries fs.DirEntr
var sharedFiles SharedFolderResponse var sharedFiles SharedFolderResponse
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err := f.rest.CallJSON(ctx, &opts, nil, &sharedFiles) resp, err := f.rest.CallJSON(ctx, &opts, nil, &sharedFiles)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, errors.Wrap(err, "couldn't list files") return nil, errors.Wrap(err, "couldn't list files")
@ -153,7 +156,7 @@ func (f *Fs) listFiles(ctx context.Context, directoryID int) (filesList *FilesLi
filesList = &FilesList{} filesList = &FilesList{}
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err := f.rest.CallJSON(ctx, &opts, &request, filesList) resp, err := f.rest.CallJSON(ctx, &opts, &request, filesList)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, errors.Wrap(err, "couldn't list files") return nil, errors.Wrap(err, "couldn't list files")
@ -181,7 +184,7 @@ func (f *Fs) listFolders(ctx context.Context, directoryID int) (foldersList *Fol
foldersList = &FoldersList{} foldersList = &FoldersList{}
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err := f.rest.CallJSON(ctx, &opts, &request, foldersList) resp, err := f.rest.CallJSON(ctx, &opts, &request, foldersList)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, errors.Wrap(err, "couldn't list folders") return nil, errors.Wrap(err, "couldn't list folders")
@ -275,7 +278,7 @@ func (f *Fs) makeFolder(ctx context.Context, leaf string, folderID int) (respons
response = &MakeFolderResponse{} response = &MakeFolderResponse{}
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err := f.rest.CallJSON(ctx, &opts, &request, response) resp, err := f.rest.CallJSON(ctx, &opts, &request, response)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, errors.Wrap(err, "couldn't create folder") return nil, errors.Wrap(err, "couldn't create folder")
@ -302,7 +305,7 @@ func (f *Fs) removeFolder(ctx context.Context, name string, folderID int) (respo
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.rest.CallJSON(ctx, &opts, request, response) resp, err = f.rest.CallJSON(ctx, &opts, request, response)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, errors.Wrap(err, "couldn't remove folder") return nil, errors.Wrap(err, "couldn't remove folder")
@ -331,7 +334,7 @@ func (f *Fs) deleteFile(ctx context.Context, url string) (response *GenericOKRes
response = &GenericOKResponse{} response = &GenericOKResponse{}
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err := f.rest.CallJSON(ctx, &opts, request, response) resp, err := f.rest.CallJSON(ctx, &opts, request, response)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
@ -358,7 +361,7 @@ func (f *Fs) moveFile(ctx context.Context, url string, folderID int, rename stri
response = &MoveFileResponse{} response = &MoveFileResponse{}
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err := f.rest.CallJSON(ctx, &opts, request, response) resp, err := f.rest.CallJSON(ctx, &opts, request, response)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
@ -383,7 +386,7 @@ func (f *Fs) copyFile(ctx context.Context, url string, folderID int, rename stri
response = &CopyFileResponse{} response = &CopyFileResponse{}
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err := f.rest.CallJSON(ctx, &opts, request, response) resp, err := f.rest.CallJSON(ctx, &opts, request, response)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
@ -405,7 +408,7 @@ func (f *Fs) getUploadNode(ctx context.Context) (response *GetUploadNodeResponse
response = &GetUploadNodeResponse{} response = &GetUploadNodeResponse{}
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err := f.rest.CallJSON(ctx, &opts, nil, response) resp, err := f.rest.CallJSON(ctx, &opts, nil, response)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, errors.Wrap(err, "didnt got an upload node") return nil, errors.Wrap(err, "didnt got an upload node")
@ -448,7 +451,7 @@ func (f *Fs) uploadFile(ctx context.Context, in io.Reader, size int64, fileName,
err = f.pacer.CallNoRetry(func() (bool, error) { err = f.pacer.CallNoRetry(func() (bool, error) {
resp, err := f.rest.CallJSON(ctx, &opts, nil, nil) resp, err := f.rest.CallJSON(ctx, &opts, nil, nil)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
@ -482,7 +485,7 @@ func (f *Fs) endUpload(ctx context.Context, uploadID string, nodeurl string) (re
response = &EndFileUploadResponse{} response = &EndFileUploadResponse{}
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err := f.rest.CallJSON(ctx, &opts, nil, response) resp, err := f.rest.CallJSON(ctx, &opts, nil, response)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {

2
backend/fichier/object.go
View File

@ -94,7 +94,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.rest.Call(ctx, &opts) resp, err = o.fs.rest.Call(ctx, &opts)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {

9
backend/filefabric/filefabric.go
View File

@ -228,7 +228,10 @@ var retryStatusCodes = []struct {
// shouldRetry returns a boolean as to whether this resp and err // shouldRetry returns a boolean as to whether this resp and err
// deserve to be retried. It returns the err as a convenience // deserve to be retried. It returns the err as a convenience
func (f *Fs) shouldRetry(resp *http.Response, err error, status api.OKError) (bool, error) { func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error, status api.OKError) (bool, error) {
if fserrors.ContextError(ctx, &err) {
return false, err
}
if err != nil { if err != nil {
return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
} }
@ -401,7 +404,7 @@ func (f *Fs) rpc(ctx context.Context, function string, p params, result api.OKEr
// Refresh the body each retry // Refresh the body each retry
opts.Body = strings.NewReader(data.Encode()) opts.Body = strings.NewReader(data.Encode())
resp, err = f.srv.CallJSON(ctx, &opts, nil, result) resp, err = f.srv.CallJSON(ctx, &opts, nil, result)
return f.shouldRetry(resp, err, result) return f.shouldRetry(ctx, resp, err, result)
}) })
if err != nil { if err != nil {
return resp, err return resp, err
@ -1277,7 +1280,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
} }
err = o.fs.pacer.CallNoRetry(func() (bool, error) { err = o.fs.pacer.CallNoRetry(func() (bool, error) {
resp, err := o.fs.srv.CallJSON(ctx, &opts, nil, &uploader) resp, err := o.fs.srv.CallJSON(ctx, &opts, nil, &uploader)
return o.fs.shouldRetry(resp, err, nil) return o.fs.shouldRetry(ctx, resp, err, nil)
}) })
if err != nil { if err != nil {
return errors.Wrap(err, "failed to upload") return errors.Wrap(err, "failed to upload")

29
backend/googlecloudstorage/googlecloudstorage.go
View File

@ -329,7 +329,10 @@ func (f *Fs) Features() *fs.Features {
} }
// shouldRetry determines whether a given err rates being retried // shouldRetry determines whether a given err rates being retried
func shouldRetry(err error) (again bool, errOut error) { func shouldRetry(ctx context.Context, err error) (again bool, errOut error) {
if fserrors.ContextError(ctx, &err) {
return false, err
}
again = false again = false
if err != nil { if err != nil {
if fserrors.ShouldRetry(err) { if fserrors.ShouldRetry(err) {
@ -455,7 +458,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
encodedDirectory := f.opt.Enc.FromStandardPath(f.rootDirectory) encodedDirectory := f.opt.Enc.FromStandardPath(f.rootDirectory)
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
_, err = f.svc.Objects.Get(f.rootBucket, encodedDirectory).Context(ctx).Do() _, err = f.svc.Objects.Get(f.rootBucket, encodedDirectory).Context(ctx).Do()
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
if err == nil { if err == nil {
newRoot := path.Dir(f.root) newRoot := path.Dir(f.root)
@ -521,7 +524,7 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
var objects *storage.Objects var objects *storage.Objects
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
objects, err = list.Context(ctx).Do() objects, err = list.Context(ctx).Do()
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
if err != nil { if err != nil {
if gErr, ok := err.(*googleapi.Error); ok { if gErr, ok := err.(*googleapi.Error); ok {
@ -624,7 +627,7 @@ func (f *Fs) listBuckets(ctx context.Context) (entries fs.DirEntries, err error)
var buckets *storage.Buckets var buckets *storage.Buckets
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
buckets, err = listBuckets.Context(ctx).Do() buckets, err = listBuckets.Context(ctx).Do()
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@ -750,7 +753,7 @@ func (f *Fs) makeBucket(ctx context.Context, bucket string) (err error) {
// service account that only has the "Storage Object Admin" role. See #2193 for details. // service account that only has the "Storage Object Admin" role. See #2193 for details.
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
_, err = f.svc.Objects.List(bucket).MaxResults(1).Context(ctx).Do() _, err = f.svc.Objects.List(bucket).MaxResults(1).Context(ctx).Do()
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
if err == nil { if err == nil {
// Bucket already exists // Bucket already exists
@ -785,7 +788,7 @@ func (f *Fs) makeBucket(ctx context.Context, bucket string) (err error) {
insertBucket.PredefinedAcl(f.opt.BucketACL) insertBucket.PredefinedAcl(f.opt.BucketACL)
} }
_, err = insertBucket.Context(ctx).Do() _, err = insertBucket.Context(ctx).Do()
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
}, nil) }, nil)
} }
@ -802,7 +805,7 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) (err error) {
return f.cache.Remove(bucket, func() error { return f.cache.Remove(bucket, func() error {
return f.pacer.Call(func() (bool, error) { return f.pacer.Call(func() (bool, error) {
err = f.svc.Buckets.Delete(bucket).Context(ctx).Do() err = f.svc.Buckets.Delete(bucket).Context(ctx).Do()
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
}) })
} }
@ -848,7 +851,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
for { for {
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
rewriteResponse, err = rewriteRequest.Context(ctx).Do() rewriteResponse, err = rewriteRequest.Context(ctx).Do()
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@ -941,7 +944,7 @@ func (o *Object) readObjectInfo(ctx context.Context) (object *storage.Object, er
bucket, bucketPath := o.split() bucket, bucketPath := o.split()
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
object, err = o.fs.svc.Objects.Get(bucket, bucketPath).Context(ctx).Do() object, err = o.fs.svc.Objects.Get(bucket, bucketPath).Context(ctx).Do()
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
if err != nil { if err != nil {
if gErr, ok := err.(*googleapi.Error); ok { if gErr, ok := err.(*googleapi.Error); ok {
@ -1012,7 +1015,7 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) (err error)
copyObject.DestinationPredefinedAcl(o.fs.opt.ObjectACL) copyObject.DestinationPredefinedAcl(o.fs.opt.ObjectACL)
} }
newObject, err = copyObject.Context(ctx).Do() newObject, err = copyObject.Context(ctx).Do()
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
if err != nil { if err != nil {
return err return err
@ -1043,7 +1046,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
_ = res.Body.Close() // ignore error _ = res.Body.Close() // ignore error
} }
} }
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@ -1109,7 +1112,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
insertObject.PredefinedAcl(o.fs.opt.ObjectACL) insertObject.PredefinedAcl(o.fs.opt.ObjectACL)
} }
newObject, err = insertObject.Context(ctx).Do() newObject, err = insertObject.Context(ctx).Do()
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
if err != nil { if err != nil {
return err return err
@ -1124,7 +1127,7 @@ func (o *Object) Remove(ctx context.Context) (err error) {
bucket, bucketPath := o.split() bucket, bucketPath := o.split()
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
err = o.fs.svc.Objects.Delete(bucket, bucketPath).Context(ctx).Do() err = o.fs.svc.Objects.Delete(bucket, bucketPath).Context(ctx).Do()
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
return err return err
} }

31
backend/googlephotos/googlephotos.go
View File

@ -240,7 +240,10 @@ var retryErrorCodes = []int{
// shouldRetry returns a boolean as to whether this resp and err // shouldRetry returns a boolean as to whether this resp and err
// deserve to be retried. It returns the err as a convenience // deserve to be retried. It returns the err as a convenience
func shouldRetry(resp *http.Response, err error) (bool, error) { func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
if fserrors.ContextError(ctx, &err) {
return false, err
}
return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
} }
@ -329,7 +332,7 @@ func (f *Fs) fetchEndpoint(ctx context.Context, name string) (endpoint string, e
var openIDconfig map[string]interface{} var openIDconfig map[string]interface{}
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err := f.unAuth.CallJSON(ctx, &opts, nil, &openIDconfig) resp, err := f.unAuth.CallJSON(ctx, &opts, nil, &openIDconfig)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return "", errors.Wrap(err, "couldn't read openID config") return "", errors.Wrap(err, "couldn't read openID config")
@ -358,7 +361,7 @@ func (f *Fs) UserInfo(ctx context.Context) (userInfo map[string]string, err erro
} }
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err := f.srv.CallJSON(ctx, &opts, nil, &userInfo) resp, err := f.srv.CallJSON(ctx, &opts, nil, &userInfo)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, errors.Wrap(err, "couldn't read user info") return nil, errors.Wrap(err, "couldn't read user info")
@ -389,7 +392,7 @@ func (f *Fs) Disconnect(ctx context.Context) (err error) {
var res interface{} var res interface{}
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err := f.srv.CallJSON(ctx, &opts, nil, &res) resp, err := f.srv.CallJSON(ctx, &opts, nil, &res)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return errors.Wrap(err, "couldn't revoke token") return errors.Wrap(err, "couldn't revoke token")
@ -476,7 +479,7 @@ func (f *Fs) listAlbums(ctx context.Context, shared bool) (all *albums, err erro
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, errors.Wrap(err, "couldn't list albums") return nil, errors.Wrap(err, "couldn't list albums")
@ -531,7 +534,7 @@ func (f *Fs) list(ctx context.Context, filter api.SearchFilter, fn listFn) (err
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, &filter, &result) resp, err = f.srv.CallJSON(ctx, &opts, &filter, &result)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return errors.Wrap(err, "couldn't list files") return errors.Wrap(err, "couldn't list files")
@ -675,7 +678,7 @@ func (f *Fs) createAlbum(ctx context.Context, albumTitle string) (album *api.Alb
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, request, &result) resp, err = f.srv.CallJSON(ctx, &opts, request, &result)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, errors.Wrap(err, "couldn't create album") return nil, errors.Wrap(err, "couldn't create album")
@ -810,7 +813,7 @@ func (o *Object) Size() int64 {
} }
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.Call(ctx, &opts) resp, err = o.fs.srv.Call(ctx, &opts)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
fs.Debugf(o, "Reading size failed: %v", err) fs.Debugf(o, "Reading size failed: %v", err)
@ -861,7 +864,7 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
var resp *http.Response var resp *http.Response
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &item) resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &item)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return errors.Wrap(err, "couldn't get media item") return errors.Wrap(err, "couldn't get media item")
@ -938,7 +941,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
} }
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.Call(ctx, &opts) resp, err = o.fs.srv.Call(ctx, &opts)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@ -993,10 +996,10 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
err = o.fs.pacer.CallNoRetry(func() (bool, error) { err = o.fs.pacer.CallNoRetry(func() (bool, error) {
resp, err = o.fs.srv.Call(ctx, &opts) resp, err = o.fs.srv.Call(ctx, &opts)
if err != nil { if err != nil {
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
} }
token, err = rest.ReadBody(resp) token, err = rest.ReadBody(resp)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return errors.Wrap(err, "couldn't upload file") return errors.Wrap(err, "couldn't upload file")
@ -1024,7 +1027,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
var result api.BatchCreateResponse var result api.BatchCreateResponse
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.CallJSON(ctx, &opts, request, &result) resp, err = o.fs.srv.CallJSON(ctx, &opts, request, &result)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return errors.Wrap(err, "failed to create media item") return errors.Wrap(err, "failed to create media item")
@ -1069,7 +1072,7 @@ func (o *Object) Remove(ctx context.Context) (err error) {
var resp *http.Response var resp *http.Response
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.CallJSON(ctx, &opts, &request, nil) resp, err = o.fs.srv.CallJSON(ctx, &opts, &request, nil)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return errors.Wrap(err, "couldn't delete item from album") return errors.Wrap(err, "couldn't delete item from album")

25
backend/jottacloud/jottacloud.go
View File

@ -235,7 +235,10 @@ var retryErrorCodes = []int{
// shouldRetry returns a boolean as to whether this resp and err // shouldRetry returns a boolean as to whether this resp and err
// deserve to be retried. It returns the err as a convenience // deserve to be retried. It returns the err as a convenience
func shouldRetry(resp *http.Response, err error) (bool, error) { func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
if fserrors.ContextError(ctx, &err) {
return false, err
}
return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
} }
@ -615,7 +618,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.Jo
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallXML(ctx, &opts, nil, &result) resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if apiErr, ok := err.(*api.Error); ok { if apiErr, ok := err.(*api.Error); ok {
@ -854,7 +857,7 @@ func (f *Fs) CreateDir(ctx context.Context, path string) (jf *api.JottaFolder, e
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallXML(ctx, &opts, nil, &jf) resp, err = f.srv.CallXML(ctx, &opts, nil, &jf)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
//fmt.Printf("...Error %v\n", err) //fmt.Printf("...Error %v\n", err)
@ -883,7 +886,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
var result api.JottaFolder var result api.JottaFolder
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallXML(ctx, &opts, nil, &result) resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
@ -995,7 +998,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
var result api.JottaFolder // Could be JottaFileDirList, but JottaFolder is close enough var result api.JottaFolder // Could be JottaFileDirList, but JottaFolder is close enough
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallXML(ctx, &opts, nil, &result) resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
if apiErr, ok := err.(*api.Error); ok { if apiErr, ok := err.(*api.Error); ok {
@ -1101,7 +1104,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.Call(ctx, &opts) resp, err = f.srv.Call(ctx, &opts)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return errors.Wrap(err, "couldn't purge directory") return errors.Wrap(err, "couldn't purge directory")
@ -1140,7 +1143,7 @@ func (f *Fs) copyOrMove(ctx context.Context, method, src, dest string) (info *ap
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallXML(ctx, &opts, nil, &info) resp, err = f.srv.CallXML(ctx, &opts, nil, &info)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@ -1268,7 +1271,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
var result api.JottaFile var result api.JottaFile
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallXML(ctx, &opts, nil, &result) resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if apiErr, ok := err.(*api.Error); ok { if apiErr, ok := err.(*api.Error); ok {
@ -1446,7 +1449,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.Call(ctx, &opts) resp, err = o.fs.srv.Call(ctx, &opts)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@ -1559,7 +1562,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
var response api.AllocateFileResponse var response api.AllocateFileResponse
err = o.fs.pacer.CallNoRetry(func() (bool, error) { err = o.fs.pacer.CallNoRetry(func() (bool, error) {
resp, err = o.fs.apiSrv.CallJSON(ctx, &opts, &request, &response) resp, err = o.fs.apiSrv.CallJSON(ctx, &opts, &request, &response)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return err return err
@ -1624,7 +1627,7 @@ func (o *Object) Remove(ctx context.Context) error {
return o.fs.pacer.Call(func() (bool, error) { return o.fs.pacer.Call(func() (bool, error) {
resp, err := o.fs.srv.CallXML(ctx, &opts, nil, nil) resp, err := o.fs.srv.CallXML(ctx, &opts, nil, nil)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
} }

41
backend/opendrive/opendrive.go
View File

@ -207,7 +207,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
Path: "/session/login.json", Path: "/session/login.json",
} }
resp, err = f.srv.CallJSON(ctx, &opts, &account, &f.session) resp, err = f.srv.CallJSON(ctx, &opts, &account, &f.session)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, errors.Wrap(err, "failed to create session") return nil, errors.Wrap(err, "failed to create session")
@ -294,7 +294,7 @@ func (f *Fs) deleteObject(ctx context.Context, id string) error {
Path: "/folder/remove.json", Path: "/folder/remove.json",
} }
resp, err := f.srv.CallJSON(ctx, &opts, &removeDirData, nil) resp, err := f.srv.CallJSON(ctx, &opts, &removeDirData, nil)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
} }
@ -389,7 +389,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
Path: "/file/move_copy.json", Path: "/file/move_copy.json",
} }
resp, err = f.srv.CallJSON(ctx, &opts, &copyFileData, &response) resp, err = f.srv.CallJSON(ctx, &opts, &copyFileData, &response)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@ -446,7 +446,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
Path: "/file/move_copy.json", Path: "/file/move_copy.json",
} }
resp, err = f.srv.CallJSON(ctx, &opts, &copyFileData, &response) resp, err = f.srv.CallJSON(ctx, &opts, &copyFileData, &response)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@ -495,7 +495,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
Path: "/folder/move_copy.json", Path: "/folder/move_copy.json",
} }
resp, err = f.srv.CallJSON(ctx, &opts, &moveFolderData, &response) resp, err = f.srv.CallJSON(ctx, &opts, &moveFolderData, &response)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
fs.Debugf(src, "DirMove error %v", err) fs.Debugf(src, "DirMove error %v", err)
@ -583,7 +583,7 @@ func (f *Fs) readMetaDataForFolderID(ctx context.Context, id string) (info *Fold
} }
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &info) resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@ -633,7 +633,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
Path: "/upload/create_file.json", Path: "/upload/create_file.json",
} }
resp, err = o.fs.srv.CallJSON(ctx, &opts, &createFileData, &response) resp, err = o.fs.srv.CallJSON(ctx, &opts, &createFileData, &response)
return o.fs.shouldRetry(resp, err) return o.fs.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, errors.Wrap(err, "failed to create file") return nil, errors.Wrap(err, "failed to create file")
@ -659,7 +659,10 @@ var retryErrorCodes = []int{
// shouldRetry returns a boolean as to whether this resp and err // shouldRetry returns a boolean as to whether this resp and err
// deserve to be retried. It returns the err as a convenience // deserve to be retried. It returns the err as a convenience
func (f *Fs) shouldRetry(resp *http.Response, err error) (bool, error) { func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
if fserrors.ContextError(ctx, &err) {
return false, err
}
return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
} }
@ -685,7 +688,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
Path: "/folder.json", Path: "/folder.json",
} }
resp, err = f.srv.CallJSON(ctx, &opts, &createDirData, &response) resp, err = f.srv.CallJSON(ctx, &opts, &createDirData, &response)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return "", err return "", err
@ -713,7 +716,7 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin
Path: "/folder/list.json/" + f.session.SessionID + "/" + pathID, Path: "/folder/list.json/" + f.session.SessionID + "/" + pathID,
} }
resp, err = f.srv.CallJSON(ctx, &opts, nil, &folderList) resp, err = f.srv.CallJSON(ctx, &opts, nil, &folderList)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return "", false, errors.Wrap(err, "failed to get folder list") return "", false, errors.Wrap(err, "failed to get folder list")
@ -756,7 +759,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
folderList := FolderList{} folderList := FolderList{}
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &folderList) resp, err = f.srv.CallJSON(ctx, &opts, nil, &folderList)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, errors.Wrap(err, "failed to get folder list") return nil, errors.Wrap(err, "failed to get folder list")
@ -845,7 +848,7 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
} }
err := o.fs.pacer.Call(func() (bool, error) { err := o.fs.pacer.Call(func() (bool, error) {
resp, err := o.fs.srv.CallJSON(ctx, &opts, &update, nil) resp, err := o.fs.srv.CallJSON(ctx, &opts, &update, nil)
return o.fs.shouldRetry(resp, err) return o.fs.shouldRetry(ctx, resp, err)
}) })
o.modTime = modTime o.modTime = modTime
@ -865,7 +868,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
var resp *http.Response var resp *http.Response
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.Call(ctx, &opts) resp, err = o.fs.srv.Call(ctx, &opts)
return o.fs.shouldRetry(resp, err) return o.fs.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, errors.Wrap(err, "failed to open file)") return nil, errors.Wrap(err, "failed to open file)")
@ -884,7 +887,7 @@ func (o *Object) Remove(ctx context.Context) error {
Path: "/file.json/" + o.fs.session.SessionID + "/" + o.id, Path: "/file.json/" + o.fs.session.SessionID + "/" + o.id,
} }
resp, err := o.fs.srv.Call(ctx, &opts) resp, err := o.fs.srv.Call(ctx, &opts)
return o.fs.shouldRetry(resp, err) return o.fs.shouldRetry(ctx, resp, err)
}) })
} }
@ -913,7 +916,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
Path: "/upload/open_file_upload.json", Path: "/upload/open_file_upload.json",
} }
resp, err := o.fs.srv.CallJSON(ctx, &opts, &openUploadData, &openResponse) resp, err := o.fs.srv.CallJSON(ctx, &opts, &openUploadData, &openResponse)
return o.fs.shouldRetry(resp, err) return o.fs.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return errors.Wrap(err, "failed to create file") return errors.Wrap(err, "failed to create file")
@ -957,7 +960,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
} }
resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &reply) resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &reply)
return o.fs.shouldRetry(resp, err) return o.fs.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return errors.Wrap(err, "failed to create file") return errors.Wrap(err, "failed to create file")
@ -980,7 +983,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
Path: "/upload/close_file_upload.json", Path: "/upload/close_file_upload.json",
} }
resp, err = o.fs.srv.CallJSON(ctx, &opts, &closeUploadData, &closeResponse) resp, err = o.fs.srv.CallJSON(ctx, &opts, &closeUploadData, &closeResponse)
return o.fs.shouldRetry(resp, err) return o.fs.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return errors.Wrap(err, "failed to create file") return errors.Wrap(err, "failed to create file")
@ -1006,7 +1009,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
Path: "/file/access.json", Path: "/file/access.json",
} }
resp, err = o.fs.srv.CallJSON(ctx, &opts, &update, nil) resp, err = o.fs.srv.CallJSON(ctx, &opts, &update, nil)
return o.fs.shouldRetry(resp, err) return o.fs.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return err return err
@ -1032,7 +1035,7 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
o.fs.session.SessionID, directoryID, url.QueryEscape(o.fs.opt.Enc.FromStandardName(leaf))), o.fs.session.SessionID, directoryID, url.QueryEscape(o.fs.opt.Enc.FromStandardName(leaf))),
} }
resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &folderList) resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &folderList)
return o.fs.shouldRetry(resp, err) return o.fs.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return errors.Wrap(err, "failed to get folder list") return errors.Wrap(err, "failed to get folder list")

35
backend/pcloud/pcloud.go
View File

@ -213,7 +213,10 @@ var retryErrorCodes = []int{
// shouldRetry returns a boolean as to whether this resp and err // shouldRetry returns a boolean as to whether this resp and err
// deserve to be retried. It returns the err as a convenience // deserve to be retried. It returns the err as a convenience
func shouldRetry(resp *http.Response, err error) (bool, error) { func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
if fserrors.ContextError(ctx, &err) {
return false, err
}
doRetry := false doRetry := false
// Check if it is an api.Error // Check if it is an api.Error
@ -405,7 +408,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
err = result.Error.Update(err) err = result.Error.Update(err)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
//fmt.Printf("...Error %v\n", err) //fmt.Printf("...Error %v\n", err)
@ -460,7 +463,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
err = result.Error.Update(err) err = result.Error.Update(err)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return found, errors.Wrap(err, "couldn't list files") return found, errors.Wrap(err, "couldn't list files")
@ -597,7 +600,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
err = result.Error.Update(err) err = result.Error.Update(err)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return errors.Wrap(err, "rmdir failed") return errors.Wrap(err, "rmdir failed")
@ -662,7 +665,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
err = result.Error.Update(err) err = result.Error.Update(err)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@ -700,7 +703,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
return f.pacer.Call(func() (bool, error) { return f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
err = result.Update(err) err = result.Update(err)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
} }
@ -740,7 +743,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
err = result.Error.Update(err) err = result.Error.Update(err)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@ -787,7 +790,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
err = result.Error.Update(err) err = result.Error.Update(err)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return err return err
@ -814,7 +817,7 @@ func (f *Fs) linkDir(ctx context.Context, dirID string, expire fs.Duration) (str
err := f.pacer.Call(func() (bool, error) { err := f.pacer.Call(func() (bool, error) {
resp, err := f.srv.CallJSON(ctx, &opts, nil, &result) resp, err := f.srv.CallJSON(ctx, &opts, nil, &result)
err = result.Error.Update(err) err = result.Error.Update(err)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return "", err return "", err
@ -838,7 +841,7 @@ func (f *Fs) linkFile(ctx context.Context, path string, expire fs.Duration) (str
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err := f.srv.CallJSON(ctx, &opts, nil, &result) resp, err := f.srv.CallJSON(ctx, &opts, nil, &result)
err = result.Error.Update(err) err = result.Error.Update(err)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return "", err return "", err
@ -869,7 +872,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &q) resp, err = f.srv.CallJSON(ctx, &opts, nil, &q)
err = q.Error.Update(err) err = q.Error.Update(err)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, errors.Wrap(err, "about failed") return nil, errors.Wrap(err, "about failed")
@ -927,7 +930,7 @@ func (o *Object) getHashes(ctx context.Context) (err error) {
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &result) resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &result)
err = result.Error.Update(err) err = result.Error.Update(err)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return err return err
@ -1046,7 +1049,7 @@ func (o *Object) downloadURL(ctx context.Context) (URL string, err error) {
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &result) resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &result)
err = result.Error.Update(err) err = result.Error.Update(err)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return "", err return "", err
@ -1072,7 +1075,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
} }
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.Call(ctx, &opts) resp, err = o.fs.srv.Call(ctx, &opts)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@ -1151,7 +1154,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
err = o.fs.pacer.CallNoRetry(func() (bool, error) { err = o.fs.pacer.CallNoRetry(func() (bool, error) {
resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &result) resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &result)
err = result.Error.Update(err) err = result.Error.Update(err)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
// sometimes pcloud leaves a half complete file on // sometimes pcloud leaves a half complete file on
@ -1181,7 +1184,7 @@ func (o *Object) Remove(ctx context.Context) error {
return o.fs.pacer.Call(func() (bool, error) { return o.fs.pacer.Call(func() (bool, error) {
resp, err := o.fs.srv.CallJSON(ctx, &opts, nil, &result) resp, err := o.fs.srv.CallJSON(ctx, &opts, nil, &result)
err = result.Error.Update(err) err = result.Error.Update(err)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
} }

25
backend/premiumizeme/premiumizeme.go
View File

@ -176,7 +176,10 @@ var retryErrorCodes = []int{
// shouldRetry returns a boolean as to whether this resp and err // shouldRetry returns a boolean as to whether this resp and err
// deserve to be retried. It returns the err as a convenience // deserve to be retried. It returns the err as a convenience
func shouldRetry(resp *http.Response, err error) (bool, error) { func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
if fserrors.ContextError(ctx, &err) {
return false, err
}
return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
} }
@ -370,7 +373,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
} }
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &info) resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
//fmt.Printf("...Error %v\n", err) //fmt.Printf("...Error %v\n", err)
@ -407,7 +410,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return found, errors.Wrap(err, "couldn't list files") return found, errors.Wrap(err, "couldn't list files")
@ -581,7 +584,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
var result api.Response var result api.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return errors.Wrap(err, "rmdir failed") return errors.Wrap(err, "rmdir failed")
@ -660,7 +663,7 @@ func (f *Fs) move(ctx context.Context, isFile bool, id, oldLeaf, newLeaf, oldDir
var result api.Response var result api.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return errors.Wrap(err, "Move http") return errors.Wrap(err, "Move http")
@ -769,7 +772,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
} }
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &info) resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, errors.Wrap(err, "CreateDir http") return nil, errors.Wrap(err, "CreateDir http")
@ -896,7 +899,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
} }
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.Call(ctx, &opts) resp, err = o.fs.srv.Call(ctx, &opts)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@ -934,7 +937,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &info) resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &info)
if err != nil { if err != nil {
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
} }
// Just check the download URL resolves - sometimes // Just check the download URL resolves - sometimes
// the URLs returned by premiumize.me don't resolve so // the URLs returned by premiumize.me don't resolve so
@ -993,7 +996,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
var result api.Response var result api.Response
err = o.fs.pacer.CallNoRetry(func() (bool, error) { err = o.fs.pacer.CallNoRetry(func() (bool, error) {
resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &result) resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &result)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return errors.Wrap(err, "upload file http") return errors.Wrap(err, "upload file http")
@ -1035,7 +1038,7 @@ func (f *Fs) renameLeaf(ctx context.Context, isFile bool, id string, newLeaf str
var result api.Response var result api.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return errors.Wrap(err, "rename http") return errors.Wrap(err, "rename http")
@ -1060,7 +1063,7 @@ func (f *Fs) remove(ctx context.Context, id string) (err error) {
var result api.Response var result api.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return errors.Wrap(err, "remove http") return errors.Wrap(err, "remove http")

6
backend/putio/error.go
View File

@ -1,6 +1,7 @@
package putio package putio
import ( import (
"context"
"fmt" "fmt"
"net/http" "net/http"
@ -29,7 +30,10 @@ func (e *statusCodeError) Temporary() bool {
// shouldRetry returns a boolean as to whether this err deserves to be // shouldRetry returns a boolean as to whether this err deserves to be
// retried. It returns the err as a convenience // retried. It returns the err as a convenience
func shouldRetry(err error) (bool, error) { func shouldRetry(ctx context.Context, err error) (bool, error) {
if fserrors.ContextError(ctx, &err) {
return false, err
}
if err == nil { if err == nil {
return false, nil return false, nil
} }

32
backend/putio/fs.go
View File

@ -147,7 +147,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
// fs.Debugf(f, "creating folder. part: %s, parentID: %d", leaf, parentID) // fs.Debugf(f, "creating folder. part: %s, parentID: %d", leaf, parentID)
entry, err = f.client.Files.CreateFolder(ctx, f.opt.Enc.FromStandardName(leaf), parentID) entry, err = f.client.Files.CreateFolder(ctx, f.opt.Enc.FromStandardName(leaf), parentID)
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
return itoa(entry.ID), err return itoa(entry.ID), err
} }
@ -164,7 +164,7 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
// fs.Debugf(f, "listing file: %d", fileID) // fs.Debugf(f, "listing file: %d", fileID)
children, _, err = f.client.Files.List(ctx, fileID) children, _, err = f.client.Files.List(ctx, fileID)
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
if err != nil { if err != nil {
if perr, ok := err.(*putio.ErrorResponse); ok && perr.Response.StatusCode == 404 { if perr, ok := err.(*putio.ErrorResponse); ok && perr.Response.StatusCode == 404 {
@ -205,7 +205,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
// fs.Debugf(f, "listing files inside List: %d", parentID) // fs.Debugf(f, "listing files inside List: %d", parentID)
children, _, err = f.client.Files.List(ctx, parentID) children, _, err = f.client.Files.List(ctx, parentID)
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
if err != nil { if err != nil {
return return
@ -271,7 +271,7 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
// fs.Debugf(f, "getting file: %d", fileID) // fs.Debugf(f, "getting file: %d", fileID)
entry, err = f.client.Files.Get(ctx, fileID) entry, err = f.client.Files.Get(ctx, fileID)
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@ -295,7 +295,7 @@ func (f *Fs) createUpload(ctx context.Context, name string, size int64, parentID
req.Header.Set("upload-metadata", fmt.Sprintf("name %s,no-torrent %s,parent_id %s,updated-at %s", b64name, b64true, b64parentID, b64modifiedAt)) req.Header.Set("upload-metadata", fmt.Sprintf("name %s,no-torrent %s,parent_id %s,updated-at %s", b64name, b64true, b64parentID, b64modifiedAt))
fs.OpenOptionAddHTTPHeaders(req.Header, options) fs.OpenOptionAddHTTPHeaders(req.Header, options)
resp, err := f.oAuthClient.Do(req) resp, err := f.oAuthClient.Do(req)
retry, err := shouldRetry(err) retry, err := shouldRetry(ctx, err)
if retry { if retry {
return true, err return true, err
} }
@ -320,7 +320,7 @@ func (f *Fs) sendUpload(ctx context.Context, location string, size int64, in io.
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
fs.Debugf(f, "Sending zero length chunk") fs.Debugf(f, "Sending zero length chunk")
_, fileID, err = f.transferChunk(ctx, location, 0, bytes.NewReader([]byte{}), 0) _, fileID, err = f.transferChunk(ctx, location, 0, bytes.NewReader([]byte{}), 0)
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
return return
} }
@ -344,13 +344,13 @@ func (f *Fs) sendUpload(ctx context.Context, location string, size int64, in io.
// Get file offset and seek to the position // Get file offset and seek to the position
offset, err := f.getServerOffset(ctx, location) offset, err := f.getServerOffset(ctx, location)
if err != nil { if err != nil {
return shouldRetry(err) return shouldRetry(ctx, err)
} }
sentBytes := offset - chunkStart sentBytes := offset - chunkStart
fs.Debugf(f, "sentBytes: %d", sentBytes) fs.Debugf(f, "sentBytes: %d", sentBytes)
_, err = chunk.Seek(sentBytes, io.SeekStart) _, err = chunk.Seek(sentBytes, io.SeekStart)
if err != nil { if err != nil {
return shouldRetry(err) return shouldRetry(ctx, err)
} }
transferOffset = offset transferOffset = offset
reqSize = chunkSize - sentBytes reqSize = chunkSize - sentBytes
@ -367,7 +367,7 @@ func (f *Fs) sendUpload(ctx context.Context, location string, size int64, in io.
offsetMismatch = true offsetMismatch = true
return true, errors.New("connection broken") return true, errors.New("connection broken")
} }
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
if err != nil { if err != nil {
return return
@ -479,7 +479,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
// fs.Debugf(f, "listing files: %d", dirID) // fs.Debugf(f, "listing files: %d", dirID)
children, _, err = f.client.Files.List(ctx, dirID) children, _, err = f.client.Files.List(ctx, dirID)
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
if err != nil { if err != nil {
return errors.Wrap(err, "Rmdir") return errors.Wrap(err, "Rmdir")
@ -493,7 +493,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
// fs.Debugf(f, "deleting file: %d", dirID) // fs.Debugf(f, "deleting file: %d", dirID)
err = f.client.Files.Delete(ctx, dirID) err = f.client.Files.Delete(ctx, dirID)
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
f.dirCache.FlushDir(dir) f.dirCache.FlushDir(dir)
return err return err
@ -552,7 +552,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (o fs.Objec
req.Header.Set("Content-Type", "application/x-www-form-urlencoded") req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
// fs.Debugf(f, "copying file (%d) to parent_id: %s", srcObj.file.ID, directoryID) // fs.Debugf(f, "copying file (%d) to parent_id: %s", srcObj.file.ID, directoryID)
_, err = f.client.Do(req, nil) _, err = f.client.Do(req, nil)
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@ -591,7 +591,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (o fs.Objec
req.Header.Set("Content-Type", "application/x-www-form-urlencoded") req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
// fs.Debugf(f, "moving file (%d) to parent_id: %s", srcObj.file.ID, directoryID) // fs.Debugf(f, "moving file (%d) to parent_id: %s", srcObj.file.ID, directoryID)
_, err = f.client.Do(req, nil) _, err = f.client.Do(req, nil)
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@ -631,7 +631,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
req.Header.Set("Content-Type", "application/x-www-form-urlencoded") req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
// fs.Debugf(f, "moving file (%s) to parent_id: %s", srcID, dstDirectoryID) // fs.Debugf(f, "moving file (%s) to parent_id: %s", srcID, dstDirectoryID)
_, err = f.client.Do(req, nil) _, err = f.client.Do(req, nil)
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
srcFs.dirCache.FlushDir(srcRemote) srcFs.dirCache.FlushDir(srcRemote)
return err return err
@ -644,7 +644,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
// fs.Debugf(f, "getting account info") // fs.Debugf(f, "getting account info")
ai, err = f.client.Account.Info(ctx) ai, err = f.client.Account.Info(ctx)
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
if err != nil { if err != nil {
return nil, errors.Wrap(err, "about failed") return nil, errors.Wrap(err, "about failed")
@ -678,6 +678,6 @@ func (f *Fs) CleanUp(ctx context.Context) (err error) {
} }
// fs.Debugf(f, "emptying trash") // fs.Debugf(f, "emptying trash")
_, err = f.client.Do(req, nil) _, err = f.client.Do(req, nil)
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
} }

10
backend/putio/object.go
View File

@ -145,7 +145,7 @@ func (o *Object) readEntry(ctx context.Context) (f *putio.File, err error) {
if perr, ok := err.(*putio.ErrorResponse); ok && perr.Response.StatusCode == 404 { if perr, ok := err.(*putio.ErrorResponse); ok && perr.Response.StatusCode == 404 {
return false, fs.ErrorObjectNotFound return false, fs.ErrorObjectNotFound
} }
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@ -220,7 +220,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
var storageURL string var storageURL string
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
storageURL, err = o.fs.client.Files.URL(ctx, o.file.ID, true) storageURL, err = o.fs.client.Files.URL(ctx, o.file.ID, true)
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
if err != nil { if err != nil {
return return
@ -231,7 +231,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, storageURL, nil) req, err := http.NewRequestWithContext(ctx, http.MethodGet, storageURL, nil)
if err != nil { if err != nil {
return shouldRetry(err) return shouldRetry(ctx, err)
} }
req.Header.Set("User-Agent", o.fs.client.UserAgent) req.Header.Set("User-Agent", o.fs.client.UserAgent)
@ -241,7 +241,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
} }
// fs.Debugf(o, "opening file: id=%d", o.file.ID) // fs.Debugf(o, "opening file: id=%d", o.file.ID)
resp, err = o.fs.httpClient.Do(req) resp, err = o.fs.httpClient.Do(req)
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
if perr, ok := err.(*putio.ErrorResponse); ok && perr.Response.StatusCode >= 400 && perr.Response.StatusCode <= 499 { if perr, ok := err.(*putio.ErrorResponse); ok && perr.Response.StatusCode >= 400 && perr.Response.StatusCode <= 499 {
_ = resp.Body.Close() _ = resp.Body.Close()
@ -283,6 +283,6 @@ func (o *Object) Remove(ctx context.Context) (err error) {
return o.fs.pacer.Call(func() (bool, error) { return o.fs.pacer.Call(func() (bool, error) {
// fs.Debugf(o, "removing file: id=%d", o.file.ID) // fs.Debugf(o, "removing file: id=%d", o.file.ID)
err = o.fs.client.Files.Delete(ctx, o.file.ID) err = o.fs.client.Files.Delete(ctx, o.file.ID)
return shouldRetry(err) return shouldRetry(ctx, err)
}) })
} }

5
backend/seafile/seafile.go
View File

@ -408,7 +408,10 @@ var retryErrorCodes = []int{
// shouldRetry returns a boolean as to whether this resp and err // shouldRetry returns a boolean as to whether this resp and err
// deserve to be retried. It returns the err as a convenience // deserve to be retried. It returns the err as a convenience
func (f *Fs) shouldRetry(resp *http.Response, err error) (bool, error) { func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
if fserrors.ContextError(ctx, &err) {
return false, err
}
// For 429 errors look at the Retry-After: header and // For 429 errors look at the Retry-After: header and
// set the retry appropriately, starting with a minimum of 1 // set the retry appropriately, starting with a minimum of 1
// second if it isn't set. // second if it isn't set.

52
backend/seafile/webapi.go
View File

@ -86,7 +86,7 @@ func (f *Fs) getServerInfo(ctx context.Context) (account *api.ServerInfo, err er
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
if resp != nil { if resp != nil {
@ -112,7 +112,7 @@ func (f *Fs) getUserAccountInfo(ctx context.Context) (account *api.AccountInfo,
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
if resp != nil { if resp != nil {
@ -139,7 +139,7 @@ func (f *Fs) getLibraries(ctx context.Context) ([]api.Library, error) {
var err error var err error
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
if resp != nil { if resp != nil {
@ -170,7 +170,7 @@ func (f *Fs) createLibrary(ctx context.Context, libraryName, password string) (l
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, &request, &result) resp, err = f.srv.CallJSON(ctx, &opts, &request, &result)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
if resp != nil { if resp != nil {
@ -197,7 +197,7 @@ func (f *Fs) deleteLibrary(ctx context.Context, libraryID string) error {
var err error var err error
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
if resp != nil { if resp != nil {
@ -228,7 +228,7 @@ func (f *Fs) decryptLibrary(ctx context.Context, libraryID, password string) err
var err error var err error
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.Call(ctx, &opts) resp, err = f.srv.Call(ctx, &opts)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
if resp != nil { if resp != nil {
@ -271,7 +271,7 @@ func (f *Fs) getDirectoryEntriesAPIv21(ctx context.Context, libraryID, dirPath s
var err error var err error
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
if resp != nil { if resp != nil {
@ -316,7 +316,7 @@ func (f *Fs) getDirectoryDetails(ctx context.Context, libraryID, dirPath string)
var err error var err error
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
if resp != nil { if resp != nil {
@ -358,7 +358,7 @@ func (f *Fs) createDir(ctx context.Context, libraryID, dirPath string) error {
var err error var err error
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.Call(ctx, &opts) resp, err = f.srv.Call(ctx, &opts)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
if resp != nil { if resp != nil {
@ -398,7 +398,7 @@ func (f *Fs) renameDir(ctx context.Context, libraryID, dirPath, newName string)
var err error var err error
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.Call(ctx, &opts) resp, err = f.srv.Call(ctx, &opts)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
if resp != nil { if resp != nil {
@ -438,7 +438,7 @@ func (f *Fs) moveDir(ctx context.Context, srcLibraryID, srcDir, srcName, dstLibr
var err error var err error
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, &request, nil) resp, err = f.srv.CallJSON(ctx, &opts, &request, nil)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
if resp != nil { if resp != nil {
@ -474,7 +474,7 @@ func (f *Fs) deleteDir(ctx context.Context, libraryID, filePath string) error {
var err error var err error
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, nil) resp, err = f.srv.CallJSON(ctx, &opts, nil, nil)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
if resp != nil { if resp != nil {
@ -505,7 +505,7 @@ func (f *Fs) getFileDetails(ctx context.Context, libraryID, filePath string) (*a
var err error var err error
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
if resp != nil { if resp != nil {
@ -539,7 +539,7 @@ func (f *Fs) deleteFile(ctx context.Context, libraryID, filePath string) error {
} }
err := f.pacer.Call(func() (bool, error) { err := f.pacer.Call(func() (bool, error) {
resp, err := f.srv.CallJSON(ctx, &opts, nil, nil) resp, err := f.srv.CallJSON(ctx, &opts, nil, nil)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return errors.Wrap(err, "failed to delete file") return errors.Wrap(err, "failed to delete file")
@ -565,7 +565,7 @@ func (f *Fs) getDownloadLink(ctx context.Context, libraryID, filePath string) (s
var err error var err error
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
if resp != nil { if resp != nil {
@ -614,7 +614,7 @@ func (f *Fs) download(ctx context.Context, url string, size int64, options ...fs
var err error var err error
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.Call(ctx, &opts) resp, err = f.srv.Call(ctx, &opts)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
if resp != nil { if resp != nil {
@ -659,7 +659,7 @@ func (f *Fs) getUploadLink(ctx context.Context, libraryID string) (string, error
var err error var err error
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
if resp != nil { if resp != nil {
@ -739,7 +739,7 @@ func (f *Fs) listShareLinks(ctx context.Context, libraryID, remote string) ([]ap
var err error var err error
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
if resp != nil { if resp != nil {
@ -777,7 +777,7 @@ func (f *Fs) createShareLink(ctx context.Context, libraryID, remote string) (*ap
var err error var err error
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, &request, &result) resp, err = f.srv.CallJSON(ctx, &opts, &request, &result)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
if resp != nil { if resp != nil {
@ -818,7 +818,7 @@ func (f *Fs) copyFile(ctx context.Context, srcLibraryID, srcPath, dstLibraryID,
var err error var err error
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, &request, &result) resp, err = f.srv.CallJSON(ctx, &opts, &request, &result)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
if resp != nil { if resp != nil {
@ -860,7 +860,7 @@ func (f *Fs) moveFile(ctx context.Context, srcLibraryID, srcPath, dstLibraryID,
var err error var err error
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, &request, &result) resp, err = f.srv.CallJSON(ctx, &opts, &request, &result)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
if resp != nil { if resp != nil {
@ -900,7 +900,7 @@ func (f *Fs) renameFile(ctx context.Context, libraryID, filePath, newname string
var err error var err error
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, &request, &result) resp, err = f.srv.CallJSON(ctx, &opts, &request, &result)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
if resp != nil { if resp != nil {
@ -938,7 +938,7 @@ func (f *Fs) emptyLibraryTrash(ctx context.Context, libraryID string) error {
var err error var err error
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, nil) resp, err = f.srv.CallJSON(ctx, &opts, nil, nil)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
if resp != nil { if resp != nil {
@ -976,7 +976,7 @@ func (f *Fs) getDirectoryEntriesAPIv2(ctx context.Context, libraryID, dirPath st
var err error var err error
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
if resp != nil { if resp != nil {
@ -1030,7 +1030,7 @@ func (f *Fs) copyFileAPIv2(ctx context.Context, srcLibraryID, srcPath, dstLibrar
var err error var err error
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.Call(ctx, &opts) resp, err = f.srv.Call(ctx, &opts)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
if resp != nil { if resp != nil {
@ -1075,7 +1075,7 @@ func (f *Fs) renameFileAPIv2(ctx context.Context, libraryID, filePath, newname s
var err error var err error
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.Call(ctx, &opts) resp, err = f.srv.Call(ctx, &opts)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
if resp != nil { if resp != nil {

25
backend/sharefile/sharefile.go
View File

@ -299,7 +299,10 @@ var retryErrorCodes = []int{
// shouldRetry returns a boolean as to whether this resp and err // shouldRetry returns a boolean as to whether this resp and err
// deserve to be retried. It returns the err as a convenience // deserve to be retried. It returns the err as a convenience
func shouldRetry(resp *http.Response, err error) (bool, error) { func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
if fserrors.ContextError(ctx, &err) {
return false, err
}
return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
} }
@ -324,7 +327,7 @@ func (f *Fs) readMetaDataForIDPath(ctx context.Context, id, path string, directo
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &item) resp, err = f.srv.CallJSON(ctx, &opts, nil, &item)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
if resp != nil && resp.StatusCode == http.StatusNotFound { if resp != nil && resp.StatusCode == http.StatusNotFound {
@ -631,7 +634,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
} }
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, &req, &info) resp, err = f.srv.CallJSON(ctx, &opts, &req, &info)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return "", errors.Wrap(err, "CreateDir") return "", errors.Wrap(err, "CreateDir")
@ -663,7 +666,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return found, errors.Wrap(err, "couldn't list files") return found, errors.Wrap(err, "couldn't list files")
@ -912,7 +915,7 @@ func (f *Fs) updateItem(ctx context.Context, id, leaf, directoryID string, modTi
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, &update, &info) resp, err = f.srv.CallJSON(ctx, &opts, &update, &info)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@ -1133,7 +1136,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
var info *api.Item var info *api.Item
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &info) resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@ -1294,7 +1297,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
var dl api.DownloadSpecification var dl api.DownloadSpecification
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &dl) resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &dl)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, errors.Wrap(err, "open: fetch download specification") return nil, errors.Wrap(err, "open: fetch download specification")
@ -1309,7 +1312,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
} }
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.Call(ctx, &opts) resp, err = o.fs.srv.Call(ctx, &opts)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, errors.Wrap(err, "open") return nil, errors.Wrap(err, "open")
@ -1365,7 +1368,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
} }
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.CallJSON(ctx, &opts, &req, &info) resp, err = o.fs.srv.CallJSON(ctx, &opts, &req, &info)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return errors.Wrap(err, "upload get specification") return errors.Wrap(err, "upload get specification")
@ -1390,7 +1393,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
var finish api.UploadFinishResponse var finish api.UploadFinishResponse
err = o.fs.pacer.CallNoRetry(func() (bool, error) { err = o.fs.pacer.CallNoRetry(func() (bool, error) {
resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &finish) resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &finish)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return errors.Wrap(err, "upload file") return errors.Wrap(err, "upload file")
@ -1426,7 +1429,7 @@ func (f *Fs) remove(ctx context.Context, id string) (err error) {
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.Call(ctx, &opts) resp, err = f.srv.Call(ctx, &opts)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return errors.Wrap(err, "remove") return errors.Wrap(err, "remove")

2
backend/sharefile/upload.go
View File

@ -155,7 +155,7 @@ func (up *largeUpload) finish(ctx context.Context) error {
err := up.f.pacer.Call(func() (bool, error) { err := up.f.pacer.Call(func() (bool, error) {
resp, err := up.f.srv.Call(ctx, &opts) resp, err := up.f.srv.Call(ctx, &opts)
if err != nil { if err != nil {
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
} }
respBody, err = rest.ReadBody(resp) respBody, err = rest.ReadBody(resp)
// retry all errors now that the multipart upload has started // retry all errors now that the multipart upload has started

33
backend/sugarsync/sugarsync.go
View File

@ -111,7 +111,7 @@ func init() {
// FIXME // FIXME
//err = f.pacer.Call(func() (bool, error) { //err = f.pacer.Call(func() (bool, error) {
resp, err = srv.CallXML(context.Background(), &opts, &authRequest, nil) resp, err = srv.CallXML(context.Background(), &opts, &authRequest, nil)
// return shouldRetry(resp, err) // return shouldRetry(ctx, resp, err)
//}) //})
if err != nil { if err != nil {
log.Fatalf("Failed to get token: %v", err) log.Fatalf("Failed to get token: %v", err)
@ -248,7 +248,10 @@ var retryErrorCodes = []int{
// shouldRetry returns a boolean as to whether this resp and err // shouldRetry returns a boolean as to whether this resp and err
// deserve to be retried. It returns the err as a convenience // deserve to be retried. It returns the err as a convenience
func shouldRetry(resp *http.Response, err error) (bool, error) { func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
if fserrors.ContextError(ctx, &err) {
return false, err
}
return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
} }
@ -288,7 +291,7 @@ func (f *Fs) readMetaDataForID(ctx context.Context, ID string) (info *api.File,
} }
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallXML(ctx, &opts, nil, &info) resp, err = f.srv.CallXML(ctx, &opts, nil, &info)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
if resp != nil && resp.StatusCode == http.StatusNotFound { if resp != nil && resp.StatusCode == http.StatusNotFound {
@ -325,7 +328,7 @@ func (f *Fs) getAuthToken(ctx context.Context) error {
} }
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallXML(ctx, &opts, &authRequest, &authResponse) resp, err = f.srv.CallXML(ctx, &opts, &authRequest, &authResponse)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return errors.Wrap(err, "failed to get authorization") return errors.Wrap(err, "failed to get authorization")
@ -373,7 +376,7 @@ func (f *Fs) getUser(ctx context.Context) (user *api.User, err error) {
} }
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallXML(ctx, &opts, nil, &user) resp, err = f.srv.CallXML(ctx, &opts, nil, &user)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, errors.Wrap(err, "failed to get user") return nil, errors.Wrap(err, "failed to get user")
@ -567,7 +570,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
} }
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallXML(ctx, &opts, mkdir, nil) resp, err = f.srv.CallXML(ctx, &opts, mkdir, nil)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return "", err return "", err
@ -618,7 +621,7 @@ OUTER:
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallXML(ctx, &opts, nil, &result) resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return found, errors.Wrap(err, "couldn't list files") return found, errors.Wrap(err, "couldn't list files")
@ -774,7 +777,7 @@ func (f *Fs) delete(ctx context.Context, isFile bool, id string, remote string,
} }
return f.pacer.Call(func() (bool, error) { return f.pacer.Call(func() (bool, error) {
resp, err := f.srv.Call(ctx, &opts) resp, err := f.srv.Call(ctx, &opts)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
} }
// Move file/dir to deleted files if not hard delete // Move file/dir to deleted files if not hard delete
@ -880,7 +883,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallXML(ctx, &opts, &copyFile, nil) resp, err = f.srv.CallXML(ctx, &opts, &copyFile, nil)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@ -934,7 +937,7 @@ func (f *Fs) moveFile(ctx context.Context, id, leaf, directoryID string) (info *
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallXML(ctx, &opts, &move, &info) resp, err = f.srv.CallXML(ctx, &opts, &move, &info)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@ -964,7 +967,7 @@ func (f *Fs) moveDir(ctx context.Context, id, leaf, directoryID string) (err err
var resp *http.Response var resp *http.Response
return f.pacer.Call(func() (bool, error) { return f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallXML(ctx, &opts, &move, nil) resp, err = f.srv.CallXML(ctx, &opts, &move, nil)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
} }
@ -1053,7 +1056,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
var info *api.File var info *api.File
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallXML(ctx, &opts, &linkFile, &info) resp, err = f.srv.CallXML(ctx, &opts, &linkFile, &info)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return "", err return "", err
@ -1182,7 +1185,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
} }
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.Call(ctx, &opts) resp, err = o.fs.srv.Call(ctx, &opts)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@ -1204,7 +1207,7 @@ func (f *Fs) createFile(ctx context.Context, pathID, leaf, mimeType string) (new
} }
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallXML(ctx, &opts, &mkdir, nil) resp, err = f.srv.CallXML(ctx, &opts, &mkdir, nil)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return "", err return "", err
@ -1262,7 +1265,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
} }
err = o.fs.pacer.CallNoRetry(func() (bool, error) { err = o.fs.pacer.CallNoRetry(func() (bool, error) {
resp, err = o.fs.srv.Call(ctx, &opts) resp, err = o.fs.srv.Call(ctx, &opts)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return errors.Wrap(err, "failed to upload file") return errors.Wrap(err, "failed to upload file")

27
backend/webdav/webdav.go
View File

@ -196,7 +196,10 @@ var retryErrorCodes = []int{
// shouldRetry returns a boolean as to whether this resp and err // shouldRetry returns a boolean as to whether this resp and err
// deserve to be retried. It returns the err as a convenience // deserve to be retried. It returns the err as a convenience
func (f *Fs) shouldRetry(resp *http.Response, err error) (bool, error) { func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
if fserrors.ContextError(ctx, &err) {
return false, err
}
// If we have a bearer token command and it has expired then refresh it // If we have a bearer token command and it has expired then refresh it
if f.opt.BearerTokenCommand != "" && resp != nil && resp.StatusCode == 401 { if f.opt.BearerTokenCommand != "" && resp != nil && resp.StatusCode == 401 {
fs.Debugf(f, "Bearer token expired: %v", err) fs.Debugf(f, "Bearer token expired: %v", err)
@ -270,7 +273,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string, depth string)
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallXML(ctx, &opts, nil, &result) resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if apiErr, ok := err.(*api.Error); ok { if apiErr, ok := err.(*api.Error); ok {
// does not exist // does not exist
@ -628,7 +631,7 @@ func (f *Fs) listAll(ctx context.Context, dir string, directoriesOnly bool, file
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallXML(ctx, &opts, nil, &result) resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
if apiErr, ok := err.(*api.Error); ok { if apiErr, ok := err.(*api.Error); ok {
@ -800,7 +803,7 @@ func (f *Fs) _dirExists(ctx context.Context, dirPath string) (exists bool) {
var err error var err error
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallXML(ctx, &opts, nil, &result) resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
return err == nil return err == nil
} }
@ -822,7 +825,7 @@ func (f *Fs) _mkdir(ctx context.Context, dirPath string) error {
} }
err := f.pacer.Call(func() (bool, error) { err := f.pacer.Call(func() (bool, error) {
resp, err := f.srv.Call(ctx, &opts) resp, err := f.srv.Call(ctx, &opts)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if apiErr, ok := err.(*api.Error); ok { if apiErr, ok := err.(*api.Error); ok {
// Check if it already exists. The response code for this isn't // Check if it already exists. The response code for this isn't
@ -911,7 +914,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
var err error var err error
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallXML(ctx, &opts, nil, nil) resp, err = f.srv.CallXML(ctx, &opts, nil, nil)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return errors.Wrap(err, "rmdir failed") return errors.Wrap(err, "rmdir failed")
@ -974,7 +977,7 @@ func (f *Fs) copyOrMove(ctx context.Context, src fs.Object, remote string, metho
} }
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.Call(ctx, &opts) resp, err = f.srv.Call(ctx, &opts)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, errors.Wrap(err, "Copy call failed") return nil, errors.Wrap(err, "Copy call failed")
@ -1070,7 +1073,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
} }
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.Call(ctx, &opts) resp, err = f.srv.Call(ctx, &opts)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return errors.Wrap(err, "DirMove MOVE call failed") return errors.Wrap(err, "DirMove MOVE call failed")
@ -1112,7 +1115,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
var err error var err error
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallXML(ctx, &opts, nil, &q) resp, err = f.srv.CallXML(ctx, &opts, nil, &q)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, errors.Wrap(err, "about call failed") return nil, errors.Wrap(err, "about call failed")
@ -1240,7 +1243,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
} }
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.Call(ctx, &opts) resp, err = o.fs.srv.Call(ctx, &opts)
return o.fs.shouldRetry(resp, err) return o.fs.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@ -1291,7 +1294,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
} }
err = o.fs.pacer.CallNoRetry(func() (bool, error) { err = o.fs.pacer.CallNoRetry(func() (bool, error) {
resp, err = o.fs.srv.Call(ctx, &opts) resp, err = o.fs.srv.Call(ctx, &opts)
return o.fs.shouldRetry(resp, err) return o.fs.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
// Give the WebDAV server a chance to get its internal state in order after the // Give the WebDAV server a chance to get its internal state in order after the
@ -1318,7 +1321,7 @@ func (o *Object) Remove(ctx context.Context) error {
} }
return o.fs.pacer.Call(func() (bool, error) { return o.fs.pacer.Call(func() (bool, error) {
resp, err := o.fs.srv.Call(ctx, &opts) resp, err := o.fs.srv.Call(ctx, &opts)
return o.fs.shouldRetry(resp, err) return o.fs.shouldRetry(ctx, resp, err)
}) })
} }

34
backend/yandex/yandex.go
View File

@ -153,7 +153,10 @@ var retryErrorCodes = []int{
// shouldRetry returns a boolean as to whether this resp and err // shouldRetry returns a boolean as to whether this resp and err
// deserve to be retried. It returns the err as a convenience // deserve to be retried. It returns the err as a convenience
func shouldRetry(resp *http.Response, err error) (bool, error) { func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
if fserrors.ContextError(ctx, &err) {
return false, err
}
return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
} }
@ -226,7 +229,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string, options *api.
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &info) resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
@ -468,7 +471,7 @@ func (f *Fs) CreateDir(ctx context.Context, path string) (err error) {
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.Call(ctx, &opts) resp, err = f.srv.Call(ctx, &opts)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
// fmt.Printf("CreateDir %q Error: %s\n", path, err.Error()) // fmt.Printf("CreateDir %q Error: %s\n", path, err.Error())
@ -543,6 +546,9 @@ func (f *Fs) waitForJob(ctx context.Context, location string) (err error) {
var body []byte var body []byte
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.Call(ctx, &opts) resp, err = f.srv.Call(ctx, &opts)
if fserrors.ContextError(ctx, &err) {
return false, err
}
if err != nil { if err != nil {
return fserrors.ShouldRetry(err), err return fserrors.ShouldRetry(err), err
} }
@ -585,6 +591,9 @@ func (f *Fs) delete(ctx context.Context, path string, hardDelete bool) (err erro
var body []byte var body []byte
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.Call(ctx, &opts) resp, err = f.srv.Call(ctx, &opts)
if fserrors.ContextError(ctx, &err) {
return false, err
}
if err != nil { if err != nil {
return fserrors.ShouldRetry(err), err return fserrors.ShouldRetry(err), err
} }
@ -658,6 +667,9 @@ func (f *Fs) copyOrMove(ctx context.Context, method, src, dst string, overwrite
var body []byte var body []byte
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.Call(ctx, &opts) resp, err = f.srv.Call(ctx, &opts)
if fserrors.ContextError(ctx, &err) {
return false, err
}
if err != nil { if err != nil {
return fserrors.ShouldRetry(err), err return fserrors.ShouldRetry(err), err
} }
@ -810,7 +822,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.Call(ctx, &opts) resp, err = f.srv.Call(ctx, &opts)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if apiErr, ok := err.(*api.ErrorResponse); ok { if apiErr, ok := err.(*api.ErrorResponse); ok {
@ -848,7 +860,7 @@ func (f *Fs) CleanUp(ctx context.Context) (err error) {
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.Call(ctx, &opts) resp, err = f.srv.Call(ctx, &opts)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
return err return err
} }
@ -865,7 +877,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
var err error var err error
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &info) resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
@ -999,7 +1011,7 @@ func (o *Object) setCustomProperty(ctx context.Context, property string, value s
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.CallJSON(ctx, &opts, &cpr, nil) resp, err = o.fs.srv.CallJSON(ctx, &opts, &cpr, nil)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
return err return err
} }
@ -1032,7 +1044,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &dl) resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &dl)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
@ -1047,7 +1059,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
} }
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.Call(ctx, &opts) resp, err = o.fs.srv.Call(ctx, &opts)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@ -1071,7 +1083,7 @@ func (o *Object) upload(ctx context.Context, in io.Reader, overwrite bool, mimeT
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &ur) resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &ur)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
@ -1089,7 +1101,7 @@ func (o *Object) upload(ctx context.Context, in io.Reader, overwrite bool, mimeT
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.Call(ctx, &opts) resp, err = o.fs.srv.Call(ctx, &opts)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
return err return err

23
backend/zoho/zoho.go
View File

@ -257,7 +257,10 @@ var retryErrorCodes = []int{
// shouldRetry returns a boolean as to whether this resp and err // shouldRetry returns a boolean as to whether this resp and err
// deserve to be retried. It returns the err as a convenience // deserve to be retried. It returns the err as a convenience
func shouldRetry(resp *http.Response, err error) (bool, error) { func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
if fserrors.ContextError(ctx, &err) {
return false, err
}
authRetry := false authRetry := false
if resp != nil && resp.StatusCode == 401 && len(resp.Header["Www-Authenticate"]) == 1 && strings.Index(resp.Header["Www-Authenticate"][0], "expired_token") >= 0 { if resp != nil && resp.StatusCode == 401 && len(resp.Header["Www-Authenticate"]) == 1 && strings.Index(resp.Header["Www-Authenticate"][0], "expired_token") >= 0 {
@ -354,7 +357,7 @@ func (f *Fs) readMetaDataForID(ctx context.Context, id string) (*api.Item, error
var err error var err error
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@ -450,7 +453,7 @@ OUTER:
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return found, errors.Wrap(err, "couldn't list files") return found, errors.Wrap(err, "couldn't list files")
@ -555,7 +558,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
} }
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, &mkdir, &info) resp, err = f.srv.CallJSON(ctx, &opts, &mkdir, &info)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
//fmt.Printf("...Error %v\n", err) //fmt.Printf("...Error %v\n", err)
@ -664,7 +667,7 @@ func (f *Fs) upload(ctx context.Context, name string, parent string, size int64,
var uploadResponse *api.UploadResponse var uploadResponse *api.UploadResponse
err = f.pacer.CallNoRetry(func() (bool, error) { err = f.pacer.CallNoRetry(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &uploadResponse) resp, err = f.srv.CallJSON(ctx, &opts, nil, &uploadResponse)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, errors.Wrap(err, "upload error") return nil, errors.Wrap(err, "upload error")
@ -746,7 +749,7 @@ func (f *Fs) deleteObject(ctx context.Context, id string) (err error) {
} }
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, &delete, nil) resp, err = f.srv.CallJSON(ctx, &opts, &delete, nil)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return errors.Wrap(err, "delete object failed") return errors.Wrap(err, "delete object failed")
@ -816,7 +819,7 @@ func (f *Fs) rename(ctx context.Context, id, name string) (item *api.Item, err e
var result *api.ItemInfo var result *api.ItemInfo
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, &rename, &result) resp, err = f.srv.CallJSON(ctx, &opts, &rename, &result)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, errors.Wrap(err, "rename failed") return nil, errors.Wrap(err, "rename failed")
@ -869,7 +872,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
var result *api.ItemList var result *api.ItemList
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, &copyFile, &result) resp, err = f.srv.CallJSON(ctx, &opts, &copyFile, &result)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, errors.Wrap(err, "couldn't copy file") return nil, errors.Wrap(err, "couldn't copy file")
@ -914,7 +917,7 @@ func (f *Fs) move(ctx context.Context, srcID, parentID string) (item *api.Item,
var result *api.ItemList var result *api.ItemList
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, &moveFile, &result) resp, err = f.srv.CallJSON(ctx, &opts, &moveFile, &result)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, errors.Wrap(err, "move failed") return nil, errors.Wrap(err, "move failed")
@ -1181,7 +1184,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
} }
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.Call(ctx, &opts) resp, err = o.fs.srv.Call(ctx, &opts)
return shouldRetry(resp, err) return shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, err return nil, err

17
fs/fserrors/error.go
View File

@ -2,6 +2,7 @@
package fserrors package fserrors
import ( import (
"context"
"fmt" "fmt"
"io" "io"
"net/http" "net/http"
@ -437,6 +438,22 @@ func ShouldRetryHTTP(resp *http.Response, retryErrorCodes []int) bool {
return false return false
} }
// ContextError checks to see if ctx is in error.
//
// If it is in error then it overwrites *perr with the context error
// if *perr was nil and returns true.
//
// Otherwise it returns false.
func ContextError(ctx context.Context, perr *error) bool {
if ctxErr := ctx.Err(); ctxErr != nil {
if *perr == nil {
*perr = ctxErr
}
return true
}
return false
}
type causer interface { type causer interface {
Cause() error Cause() error
} }

19
fs/fserrors/error_test.go
View File

@ -1,6 +1,7 @@
package fserrors package fserrors
import ( import (
"context"
"fmt" "fmt"
"io" "io"
"net" "net"
@ -170,3 +171,21 @@ func TestRetryAfter(t *testing.T) {
assert.True(t, IsRetryAfterError(err)) assert.True(t, IsRetryAfterError(err))
assert.Contains(t, e.Error(), "try again after") assert.Contains(t, e.Error(), "try again after")
} }
func TestContextError(t *testing.T) {
var err = io.EOF
ctx, cancel := context.WithCancel(context.Background())
assert.False(t, ContextError(ctx, &err))
assert.Equal(t, io.EOF, err)
cancel()
assert.True(t, ContextError(ctx, &err))
assert.Equal(t, io.EOF, err)
err = nil
assert.True(t, ContextError(ctx, &err))
assert.Equal(t, context.Canceled, err)
}

2
fs/operations/check.go
View File

@ -311,7 +311,7 @@ func CheckEqualReaders(in1, in2 io.Reader) (differ bool, err error) {
// it returns true if differences were found // it returns true if differences were found
func CheckIdenticalDownload(ctx context.Context, dst, src fs.Object) (differ bool, err error) { func CheckIdenticalDownload(ctx context.Context, dst, src fs.Object) (differ bool, err error) {
ci := fs.GetConfig(ctx) ci := fs.GetConfig(ctx)
err = Retry(src, ci.LowLevelRetries, func() error { err = Retry(ctx, src, ci.LowLevelRetries, func() error {
differ, err = checkIdenticalDownload(ctx, dst, src) differ, err = checkIdenticalDownload(ctx, dst, src)
return err return err
}) })

8
fs/operations/operations.go
View File

@ -485,6 +485,9 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj
break break
} }
// Retry if err returned a retry error // Retry if err returned a retry error
if fserrors.ContextError(ctx, &err) {
break
}
var retry bool var retry bool
if fserrors.IsRetryError(err) || fserrors.ShouldRetry(err) { if fserrors.IsRetryError(err) || fserrors.ShouldRetry(err) {
retry = true retry = true
@ -800,7 +803,7 @@ func SameDir(fdst, fsrc fs.Info) bool {
} }
// Retry runs fn up to maxTries times if it returns a retriable error // Retry runs fn up to maxTries times if it returns a retriable error
func Retry(o interface{}, maxTries int, fn func() error) (err error) { func Retry(ctx context.Context, o interface{}, maxTries int, fn func() error) (err error) {
for tries := 1; tries <= maxTries; tries++ { for tries := 1; tries <= maxTries; tries++ {
// Call the function which might error // Call the function which might error
err = fn() err = fn()
@ -808,6 +811,9 @@ func Retry(o interface{}, maxTries int, fn func() error) (err error) {
break break
} }
// Retry if err returned a retry error // Retry if err returned a retry error
if fserrors.ContextError(ctx, &err) {
break
}
if fserrors.IsRetryError(err) || fserrors.ShouldRetry(err) { if fserrors.IsRetryError(err) || fserrors.ShouldRetry(err) {
fs.Debugf(o, "Received error: %v - low level retry %d/%d", err, tries, maxTries) fs.Debugf(o, "Received error: %v - low level retry %d/%d", err, tries, maxTries)
continue continue

8
fs/operations/operations_test.go
View File

@ -405,6 +405,8 @@ func TestDelete(t *testing.T) {
} }
func TestRetry(t *testing.T) { func TestRetry(t *testing.T) {
ctx := context.Background()
var i int var i int
var err error var err error
fn := func() error { fn := func() error {
@ -416,15 +418,15 @@ func TestRetry(t *testing.T) {
} }
i, err = 3, io.EOF i, err = 3, io.EOF
assert.Equal(t, nil, operations.Retry(nil, 5, fn)) assert.Equal(t, nil, operations.Retry(ctx, nil, 5, fn))
assert.Equal(t, 0, i) assert.Equal(t, 0, i)
i, err = 10, io.EOF i, err = 10, io.EOF
assert.Equal(t, io.EOF, operations.Retry(nil, 5, fn)) assert.Equal(t, io.EOF, operations.Retry(ctx, nil, 5, fn))
assert.Equal(t, 5, i) assert.Equal(t, 5, i)
i, err = 10, fs.ErrorObjectNotFound i, err = 10, fs.ErrorObjectNotFound
assert.Equal(t, fs.ErrorObjectNotFound, operations.Retry(nil, 5, fn)) assert.Equal(t, fs.ErrorObjectNotFound, operations.Retry(ctx, nil, 5, fn))
assert.Equal(t, 9, i) assert.Equal(t, 9, i)
} }