community
/
rclone
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Make NewObject return an error 

* make it return an error
  * make a canonical error fs.ErrorNotFound
  * make a test for it
  * remove logs/debugs of error
This commit is contained in:
Nick Craig-Wood 2016-06-25 21:23:20 +01:00
parent b1f131964e
commit ab43005422
13 changed files with 268 additions and 208 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
amazonclouddrive/amazonclouddrive.go
View File

@ -215,10 +215,13 @@ func NewFs(name, root string) (fs.Fs, error) {
// No root so return old f // No root so return old f
return f, nil return f, nil
} }
obj := newF.newObjectWithInfo(remote, nil) _, err := newF.newObjectWithInfo(remote, nil)
if obj == nil { if err != nil {
// File doesn't exist so return old f if err == fs.ErrorObjectNotFound {
return f, nil // File doesn't exist so return old f
return f, nil
}
return nil, err
} }
// return an error with an fs which points to the parent // return an error with an fs which points to the parent
return &newF, fs.ErrorIsFile return &newF, fs.ErrorIsFile
@ -228,8 +231,8 @@ func NewFs(name, root string) (fs.Fs, error) {
// Return an Object from a path // Return an Object from a path
// //
// May return nil if an error occurred // If it can't be found it returns the error fs.ErrorObjectNotFound.
func (f *Fs) newObjectWithInfo(remote string, info *acd.Node) fs.Object { func (f *Fs) newObjectWithInfo(remote string, info *acd.Node) (fs.Object, error) {
o := &Object{ o := &Object{
fs: f, fs: f,
remote: remote, remote: remote,
@ -240,17 +243,15 @@ func (f *Fs) newObjectWithInfo(remote string, info *acd.Node) fs.Object {
} else { } else {
err := o.readMetaData() // reads info and meta, returning an error err := o.readMetaData() // reads info and meta, returning an error
if err != nil { if err != nil {
fs.Log(o, "Failed to read metadata: %s", err) return nil, err
return nil
} }
} }
return o return o, nil
} }
// NewObject returns an Object from a path // NewObject finds the Object at remote. If it can't be found
// // it returns the error fs.ErrorObjectNotFound.
// May return nil if an error occurred func (f *Fs) NewObject(remote string) (fs.Object, error) {
func (f *Fs) NewObject(remote string) fs.Object {
return f.newObjectWithInfo(remote, nil) return f.newObjectWithInfo(remote, nil)
} }
@ -384,10 +385,13 @@ func (f *Fs) ListDir(out fs.ListOpts, job dircache.ListDirJob) (jobs []dircache.
} }
} }
case fileKind: case fileKind:
if o := f.newObjectWithInfo(remote, node); o != nil { o, err := f.newObjectWithInfo(remote, node)
if out.Add(o) { if err != nil {
return true out.SetError(err)
} return true
}
if out.Add(o) {
return true
} }
default: default:
// ignore ASSET etc // ignore ASSET etc
@ -430,7 +434,7 @@ func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) {
switch err { switch err {
case nil: case nil:
return o, o.Update(in, src) return o, o.Update(in, src)
case fs.ErrorDirNotFound, acd.ErrorNodeNotFound: case fs.ErrorObjectNotFound:
// Not found so create it // Not found so create it
default: default:
return nil, err return nil, err
@ -608,12 +612,17 @@ func (o *Object) Size() int64 {
// readMetaData gets the metadata if it hasn't already been fetched // readMetaData gets the metadata if it hasn't already been fetched
// //
// it also sets the info // it also sets the info
//
// If it can't be found it returns the error fs.ErrorObjectNotFound.
func (o *Object) readMetaData() (err error) { func (o *Object) readMetaData() (err error) {
if o.info != nil { if o.info != nil {
return nil return nil
} }
leaf, directoryID, err := o.fs.dirCache.FindPath(o.remote, false) leaf, directoryID, err := o.fs.dirCache.FindPath(o.remote, false)
if err != nil { if err != nil {
if err == fs.ErrorDirNotFound {
return fs.ErrorObjectNotFound
}
return err return err
} }
folder := acd.FolderFromId(directoryID, o.fs.c.Nodes) folder := acd.FolderFromId(directoryID, o.fs.c.Nodes)
@ -624,6 +633,9 @@ func (o *Object) readMetaData() (err error) {
return o.fs.shouldRetry(resp, err) return o.fs.shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
if err == acd.ErrorNodeNotFound {
return fs.ErrorObjectNotFound
}
return err return err
} }
o.info = info.Node o.info = info.Node

49
b2/b2.go
View File

@ -237,12 +237,17 @@ func NewFs(name, root string) (fs.Fs, error) {
} else { } else {
f.root += "/" f.root += "/"
} }
obj := f.NewObject(remote) _, err := f.NewObject(remote)
if obj != nil { if err != nil {
// return an error with an fs which points to the parent if err == fs.ErrorObjectNotFound {
return f, fs.ErrorIsFile // File doesn't exist so return old f
f.root = oldRoot
return f, nil
}
return nil, err
} }
f.root = oldRoot // return an error with an fs which points to the parent
return f, fs.ErrorIsFile
} }
return f, nil return f, nil
} }
@ -321,8 +326,8 @@ func (f *Fs) clearUploadURL() {
// Return an Object from a path // Return an Object from a path
// //
// May return nil if an error occurred // If it can't be found it returns the error fs.ErrorObjectNotFound.
func (f *Fs) newObjectWithInfo(remote string, info *api.File) fs.Object { func (f *Fs) newObjectWithInfo(remote string, info *api.File) (fs.Object, error) {
o := &Object{ o := &Object{
fs: f, fs: f,
remote: remote, remote: remote,
@ -330,23 +335,20 @@ func (f *Fs) newObjectWithInfo(remote string, info *api.File) fs.Object {
if info != nil { if info != nil {
err := o.decodeMetaData(info) err := o.decodeMetaData(info)
if err != nil { if err != nil {
fs.Debug(o, "Failed to decode metadata: %s", err) return nil, err
return nil
} }
} else { } else {
err := o.readMetaData() // reads info and headers, returning an error err := o.readMetaData() // reads info and headers, returning an error
if err != nil { if err != nil {
fs.Debug(o, "Failed to read metadata: %s", err) return nil, err
return nil
} }
} }
return o return o, nil
} }
// NewObject returns an Object from a path // NewObject finds the Object at remote. If it can't be found
// // it returns the error fs.ErrorObjectNotFound.
// May return nil if an error occurred func (f *Fs) NewObject(remote string) (fs.Object, error) {
func (f *Fs) NewObject(remote string) fs.Object {
return f.newObjectWithInfo(remote, nil) return f.newObjectWithInfo(remote, nil)
} }
@ -495,10 +497,12 @@ func (f *Fs) listFiles(out fs.ListOpts, dir string) {
return fs.ErrorListAborted return fs.ErrorListAborted
} }
} else { } else {
if o := f.newObjectWithInfo(remote, object); o != nil { o, err := f.newObjectWithInfo(remote, object)
if out.Add(o) { if err != nil {
return fs.ErrorListAborted return err
} }
if out.Add(o) {
return fs.ErrorListAborted
} }
} }
return nil return nil
@ -855,10 +859,13 @@ func (o *Object) readMetaData() (err error) {
return errEndList // read only 1 item return errEndList // read only 1 item
}) })
if err != nil { if err != nil {
if err == fs.ErrorDirNotFound {
return fs.ErrorObjectNotFound
}
return err return err
} }
if info == nil { if info == nil {
return errors.Errorf("object %q not found", o.remote) return fs.ErrorObjectNotFound
} }
return o.decodeMetaData(info) return o.decodeMetaData(info)
} }

64
drive/drive.go
View File

@ -80,7 +80,6 @@ var (
"text/plain": "txt", "text/plain": "txt",
} }
extensionToMimeType map[string]string extensionToMimeType map[string]string
errorObjectNotFound = errors.New("Object not found")
) )
// Register with Fs // Register with Fs
@ -337,7 +336,7 @@ func NewFs(name, path string) (fs.Fs, error) {
// No root so return old f // No root so return old f
return f, nil return f, nil
} }
_, err := newF.newObjectWithInfoErr(remote, nil) _, err := newF.newObjectWithInfo(remote, nil)
if err != nil { if err != nil {
// File doesn't exist so return old f // File doesn't exist so return old f
return f, nil return f, nil
@ -350,7 +349,9 @@ func NewFs(name, path string) (fs.Fs, error) {
} }
// Return an Object from a path // Return an Object from a path
func (f *Fs) newObjectWithInfoErr(remote string, info *drive.File) (fs.Object, error) { //
// If it can't be found it returns the error fs.ErrorObjectNotFound.
func (f *Fs) newObjectWithInfo(remote string, info *drive.File) (fs.Object, error) {
o := &Object{ o := &Object{
fs: f, fs: f,
remote: remote, remote: remote,
@ -366,21 +367,9 @@ func (f *Fs) newObjectWithInfoErr(remote string, info *drive.File) (fs.Object, e
return o, nil return o, nil
} }
// Return an Object from a path // NewObject finds the Object at remote. If it can't be found
// // it returns the error fs.ErrorObjectNotFound.
// May return nil if an error occurred func (f *Fs) NewObject(remote string) (fs.Object, error) {
func (f *Fs) newObjectWithInfo(remote string, info *drive.File) fs.Object {
o, err := f.newObjectWithInfoErr(remote, info)
if err != nil {
fs.Log(o, "Failed to read metadata: %v", err)
}
return o
}
// NewObject returns an Object from a path
//
// May return nil if an error occurred
func (f *Fs) NewObject(remote string) fs.Object {
return f.newObjectWithInfo(remote, nil) return f.newObjectWithInfo(remote, nil)
} }
@ -478,10 +467,13 @@ func (f *Fs) ListDir(out fs.ListOpts, job dircache.ListDirJob) (jobs []dircache.
} }
case item.Md5Checksum != "": case item.Md5Checksum != "":
// If item has MD5 sum it is a file stored on drive // If item has MD5 sum it is a file stored on drive
if o := f.newObjectWithInfo(remote, item); o != nil { o, err := f.newObjectWithInfo(remote, item)
if out.Add(o) { if err != nil {
return true out.SetError(err)
} return true
}
if out.Add(o) {
return true
} }
case len(item.ExportLinks) != 0: case len(item.ExportLinks) != 0:
// If item has export links then it is a google doc // If item has export links then it is a google doc
@ -489,14 +481,17 @@ func (f *Fs) ListDir(out fs.ListOpts, job dircache.ListDirJob) (jobs []dircache.
if extension == "" { if extension == "" {
fs.Debug(remote, "No export formats found") fs.Debug(remote, "No export formats found")
} else { } else {
if o := f.newObjectWithInfo(remote+"."+extension, item); o != nil { o, err := f.newObjectWithInfo(remote+"."+extension, item)
obj := o.(*Object) if err != nil {
obj.isDocument = true out.SetError(err)
obj.url = link return true
obj.bytes = -1 }
if out.Add(o) { obj := o.(*Object)
return true obj.isDocument = true
} obj.url = link
obj.bytes = -1
if out.Add(o) {
return true
} }
} }
default: default:
@ -547,11 +542,11 @@ func (f *Fs) createFileInfo(remote string, modTime time.Time, size int64) (*Obje
// //
// The new object may have been created if an error is returned // The new object may have been created if an error is returned
func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) { func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) {
exisitingObj, err := f.newObjectWithInfoErr(src.Remote(), nil) exisitingObj, err := f.newObjectWithInfo(src.Remote(), nil)
switch err { switch err {
case nil: case nil:
return exisitingObj, exisitingObj.Update(in, src) return exisitingObj, exisitingObj.Update(in, src)
case fs.ErrorDirNotFound, errorObjectNotFound: case fs.ErrorObjectNotFound:
// Not found so create it // Not found so create it
return f.PutUnchecked(in, src) return f.PutUnchecked(in, src)
default: default:
@ -857,6 +852,9 @@ func (o *Object) readMetaData() (err error) {
leaf, directoryID, err := o.fs.dirCache.FindPath(o.remote, false) leaf, directoryID, err := o.fs.dirCache.FindPath(o.remote, false)
if err != nil { if err != nil {
if err == fs.ErrorDirNotFound {
return fs.ErrorObjectNotFound
}
return err return err
} }
@ -871,7 +869,7 @@ func (o *Object) readMetaData() (err error) {
return err return err
} }
if !found { if !found {
return errorObjectNotFound return fs.ErrorObjectNotFound
} }
return nil return nil
} }

57
dropbox/dropbox.go
View File

@ -13,6 +13,7 @@ import (
"io" "io"
"io/ioutil" "io/ioutil"
"log" "log"
"net/http"
"path" "path"
"regexp" "regexp"
"strings" "strings"
@ -198,8 +199,8 @@ func (f *Fs) setRoot(root string) {
// Return an Object from a path // Return an Object from a path
// //
// May return nil if an error occurred // If it can't be found it returns the error fs.ErrorObjectNotFound.
func (f *Fs) newObjectWithInfo(remote string, info *dropbox.Entry) fs.Object { func (f *Fs) newObjectWithInfo(remote string, info *dropbox.Entry) (fs.Object, error) {
o := &Object{ o := &Object{
fs: f, fs: f,
remote: remote, remote: remote,
@ -209,17 +210,15 @@ func (f *Fs) newObjectWithInfo(remote string, info *dropbox.Entry) fs.Object {
} else { } else {
err := o.readEntryAndSetMetadata() err := o.readEntryAndSetMetadata()
if err != nil { if err != nil {
// logged already fs.Debug("Failed to read info: %s", err) return nil, err
return nil
} }
} }
return o return o, nil
} }
// NewObject returns an Object from a path // NewObject finds the Object at remote. If it can't be found
// // it returns the error fs.ErrorObjectNotFound.
// May return nil if an error occurred func (f *Fs) NewObject(remote string) (fs.Object, error) {
func (f *Fs) NewObject(remote string) fs.Object {
return f.newObjectWithInfo(remote, nil) return f.newObjectWithInfo(remote, nil)
} }
@ -322,10 +321,13 @@ func (f *Fs) list(out fs.ListOpts, dir string) {
out.SetError(err) out.SetError(err)
return return
} }
if o := f.newObjectWithInfo(path, entry); o != nil { o, err := f.newObjectWithInfo(path, entry)
if out.Add(o) { if err != nil {
return out.SetError(err)
} return
}
if out.Add(o) {
return
} }
} else { } else {
nameTree.PutFile(parentPath, lastComponent, entry) nameTree.PutFile(parentPath, lastComponent, entry)
@ -345,10 +347,12 @@ func (f *Fs) list(out fs.ListOpts, dir string) {
if err != nil { if err != nil {
return err return err
} }
if o := f.newObjectWithInfo(path, entry); o != nil { o, err := f.newObjectWithInfo(path, entry)
if out.Add(o) { if err != nil {
return fs.ErrorListAborted return err
} }
if out.Add(o) {
return fs.ErrorListAborted
} }
return nil return nil
} }
@ -387,10 +391,13 @@ func (f *Fs) listOneLevel(out fs.ListOpts, dir string) {
return return
} }
} else { } else {
if o := f.newObjectWithInfo(remote, entry); o != nil { o, err := f.newObjectWithInfo(remote, entry)
if out.Add(o) { if err != nil {
return out.SetError(err)
} return
}
if out.Add(o) {
return
} }
} }
} }
@ -622,8 +629,12 @@ func (o *Object) setMetadataFromEntry(info *dropbox.Entry) {
func (o *Object) readEntry() (*dropbox.Entry, error) { func (o *Object) readEntry() (*dropbox.Entry, error) {
entry, err := o.fs.db.Metadata(o.remotePath(), false, false, "", "", metadataLimit) entry, err := o.fs.db.Metadata(o.remotePath(), false, false, "", "", metadataLimit)
if err != nil { if err != nil {
fs.Debug(o, "Error reading file: %s", err) if dropboxErr, ok := err.(*dropbox.Error); ok {
return nil, errors.Wrap(err, "error reading file") if dropboxErr.StatusCode == http.StatusNotFound {
return nil, fs.ErrorObjectNotFound
}
}
return nil, err
} }
return entry, nil return entry, nil
} }

6
fs/fs.go
View File

@ -39,6 +39,7 @@ var (
ErrorDirExists = errors.New("can't copy directory - destination already exists") ErrorDirExists = errors.New("can't copy directory - destination already exists")
ErrorCantSetModTime = errors.New("can't set modified time") ErrorCantSetModTime = errors.New("can't set modified time")
ErrorDirNotFound = errors.New("directory not found") ErrorDirNotFound = errors.New("directory not found")
ErrorObjectNotFound = errors.New("object not found")
ErrorLevelNotSupported = errors.New("level value not supported") ErrorLevelNotSupported = errors.New("level value not supported")
ErrorListAborted = errors.New("list aborted") ErrorListAborted = errors.New("list aborted")
ErrorListOnlyRoot = errors.New("can only list from root") ErrorListOnlyRoot = errors.New("can only list from root")
@ -117,8 +118,9 @@ type Fs interface {
Info Info
ListFser ListFser
// NewObject finds the Object at remote. Returns nil if can't be found // NewObject finds the Object at remote. If it can't be found
NewObject(remote string) Object // it returns the error ErrorObjectNotFound.
NewObject(remote string) (Object, error)
// Put in to the remote path with the modTime given of the given size // Put in to the remote path with the modTime given of the given size
// //

26
fstest/fstests/fstests.go
View File

@ -171,23 +171,29 @@ func TestFsListDirEmpty(t *testing.T) {
// TestFsNewObjectNotFound tests not finding a object // TestFsNewObjectNotFound tests not finding a object
func TestFsNewObjectNotFound(t *testing.T) { func TestFsNewObjectNotFound(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
if remote.NewObject("potato") != nil { // Object in an existing directory
t.Fatal("Didn't expect to find object") o, err := remote.NewObject("potato")
} assert.Nil(t, o)
assert.Equal(t, fs.ErrorObjectNotFound, err)
// Now try an object in a non existing directory
o, err = remote.NewObject("directory/not/found/potato")
assert.Nil(t, o)
assert.Equal(t, fs.ErrorObjectNotFound, err)
} }
func findObject(t *testing.T, Name string) fs.Object { func findObject(t *testing.T, Name string) fs.Object {
var obj fs.Object var obj fs.Object
var err error
for i := 1; i <= eventualConsistencyRetries; i++ { for i := 1; i <= eventualConsistencyRetries; i++ {
obj = remote.NewObject(Name) obj, err = remote.NewObject(Name)
if obj != nil { if err == nil {
break break
} }
t.Logf("Sleeping for 1 second for findObject eventual consistency: %d/%d", i, eventualConsistencyRetries) t.Logf("Sleeping for 1 second for findObject eventual consistency: %d/%d (%v)", i, eventualConsistencyRetries, err)
time.Sleep(1 * time.Second) time.Sleep(1 * time.Second)
} }
if obj == nil { if err != nil {
t.Fatalf("Object not found: %q", Name) t.Fatalf("Object %q not found: %v", Name, err)
} }
return obj return obj
} }
@ -629,10 +635,6 @@ func TestFsIsFileNotFound(t *testing.T) {
t.Fatalf("Failed to make remote %q: %v", remoteName, err) t.Fatalf("Failed to make remote %q: %v", remoteName, err)
} }
fstest.CheckListing(t, fileRemote, []fstest.Item{}) fstest.CheckListing(t, fileRemote, []fstest.Item{})
_, ok := fileRemote.(*fs.Limited)
if ok {
t.Errorf("%v is is a fs.Limited", fileRemote)
}
} }
// TestObjectRemove tests Remove // TestObjectRemove tests Remove

32
googlecloudstorage/googlecloudstorage.go
View File

@ -267,8 +267,8 @@ func NewFs(name, root string) (fs.Fs, error) {
// Return an Object from a path // Return an Object from a path
// //
// May return nil if an error occurred // If it can't be found it returns the error fs.ErrorObjectNotFound.
func (f *Fs) newObjectWithInfo(remote string, info *storage.Object) fs.Object { func (f *Fs) newObjectWithInfo(remote string, info *storage.Object) (fs.Object, error) {
o := &Object{ o := &Object{
fs: f, fs: f,
remote: remote, remote: remote,
@ -278,17 +278,15 @@ func (f *Fs) newObjectWithInfo(remote string, info *storage.Object) fs.Object {
} else { } else {
err := o.readMetaData() // reads info and meta, returning an error err := o.readMetaData() // reads info and meta, returning an error
if err != nil { if err != nil {
// logged already FsDebug("Failed to read info: %s", err) return nil, err
return nil
} }
} }
return o return o, nil
} }
// NewObject returns an Object from a path // NewObject finds the Object at remote. If it can't be found
// // it returns the error fs.ErrorObjectNotFound.
// May return nil if an error occurred func (f *Fs) NewObject(remote string) (fs.Object, error) {
func (f *Fs) NewObject(remote string) fs.Object {
return f.newObjectWithInfo(remote, nil) return f.newObjectWithInfo(remote, nil)
} }
@ -369,10 +367,12 @@ func (f *Fs) listFiles(out fs.ListOpts, dir string) {
return fs.ErrorListAborted return fs.ErrorListAborted
} }
} else { } else {
if o := f.newObjectWithInfo(remote, object); o != nil { o, err := f.newObjectWithInfo(remote, object)
if out.Add(o) { if err != nil {
return fs.ErrorListAborted return err
} }
if out.Add(o) {
return fs.ErrorListAborted
} }
} }
return nil return nil
@ -596,7 +596,11 @@ func (o *Object) readMetaData() (err error) {
} }
object, err := o.fs.svc.Objects.Get(o.fs.bucket, o.fs.root+o.remote).Do() object, err := o.fs.svc.Objects.Get(o.fs.bucket, o.fs.root+o.remote).Do()
if err != nil { if err != nil {
fs.Debug(o, "Failed to read info: %s", err) if gErr, ok := err.(*googleapi.Error); ok {
if gErr.Code == http.StatusNotFound {
return fs.ErrorObjectNotFound
}
}
return err return err
} }
o.setMetaData(object) o.setMetaData(object)

35
local/local.go
View File

@ -76,12 +76,7 @@ func NewFs(name, root string) (fs.Fs, error) {
fi, err := os.Lstat(f.root) fi, err := os.Lstat(f.root)
if err == nil && fi.Mode().IsRegular() { if err == nil && fi.Mode().IsRegular() {
// It is a file, so use the parent as the root // It is a file, so use the parent as the root
var remote string f.root, _ = getDirFile(f.root)
f.root, remote = getDirFile(f.root)
obj := f.NewObject(remote)
if obj == nil {
return nil, errors.Errorf("failed to make object for %q in %q", remote, f.root)
}
// return an error with an fs which points to the parent // return an error with an fs which points to the parent
return f, fs.ErrorIsFile return f, fs.ErrorIsFile
} }
@ -118,24 +113,25 @@ func (f *Fs) newObject(remote string) *Object {
// Return an Object from a path // Return an Object from a path
// //
// May return nil if an error occurred // May return nil if an error occurred
func (f *Fs) newObjectWithInfo(remote string, info os.FileInfo) fs.Object { func (f *Fs) newObjectWithInfo(remote string, info os.FileInfo) (fs.Object, error) {
o := f.newObject(remote) o := f.newObject(remote)
if info != nil { if info != nil {
o.info = info o.info = info
} else { } else {
err := o.lstat() err := o.lstat()
if err != nil { if err != nil {
fs.Debug(o, "Failed to stat %s: %s", o.path, err) if os.IsNotExist(err) {
return nil return nil, fs.ErrorObjectNotFound
}
return nil, err
} }
} }
return o return o, nil
} }
// NewObject returns an Object from a path // NewObject finds the Object at remote. If it can't be found
// // it returns the error ErrorObjectNotFound.
// May return nil if an error occurred func (f *Fs) NewObject(remote string) (fs.Object, error) {
func (f *Fs) NewObject(remote string) fs.Object {
return f.newObjectWithInfo(remote, nil) return f.newObjectWithInfo(remote, nil)
} }
@ -192,10 +188,13 @@ func (f *Fs) list(out fs.ListOpts, remote string, dirpath string, level int) (su
} }
} }
} else { } else {
if fso := f.newObjectWithInfo(newRemote, fi); fso != nil { fso, err := f.newObjectWithInfo(newRemote, fi)
if fso.Storable() && out.Add(fso) { if err != nil {
return nil out.SetError(err)
} return nil
}
if fso.Storable() && out.Add(fso) {
return nil
} }
} }
} }

44
onedrive/onedrive.go
View File

@ -206,10 +206,13 @@ func NewFs(name, root string) (fs.Fs, error) {
// No root so return old f // No root so return old f
return f, nil return f, nil
} }
obj := newF.newObjectWithInfo(remote, nil) _, err := newF.newObjectWithInfo(remote, nil)
if obj == nil { if err != nil {
// File doesn't exist so return old f if err == fs.ErrorObjectNotFound {
return f, nil // File doesn't exist so return old f
return f, nil
}
return nil, err
} }
// return an error with an fs which points to the parent // return an error with an fs which points to the parent
return &newF, fs.ErrorIsFile return &newF, fs.ErrorIsFile
@ -227,8 +230,8 @@ func (f *Fs) rootSlash() string {
// Return an Object from a path // Return an Object from a path
// //
// May return nil if an error occurred // If it can't be found it returns the error fs.ErrorObjectNotFound.
func (f *Fs) newObjectWithInfo(remote string, info *api.Item) fs.Object { func (f *Fs) newObjectWithInfo(remote string, info *api.Item) (fs.Object, error) {
o := &Object{ o := &Object{
fs: f, fs: f,
remote: remote, remote: remote,
@ -239,17 +242,15 @@ func (f *Fs) newObjectWithInfo(remote string, info *api.Item) fs.Object {
} else { } else {
err := o.readMetaData() // reads info and meta, returning an error err := o.readMetaData() // reads info and meta, returning an error
if err != nil { if err != nil {
// logged already FsDebug("Failed to read info: %s", err) return nil, err
return nil
} }
} }
return o return o, nil
} }
// NewObject returns an Object from a path // NewObject finds the Object at remote. If it can't be found
// // it returns the error fs.ErrorObjectNotFound.
// May return nil if an error occurred func (f *Fs) NewObject(remote string) (fs.Object, error) {
func (f *Fs) NewObject(remote string) fs.Object {
return f.newObjectWithInfo(remote, nil) return f.newObjectWithInfo(remote, nil)
} }
@ -391,10 +392,13 @@ func (f *Fs) ListDir(out fs.ListOpts, job dircache.ListDirJob) (jobs []dircache.
} }
} }
} else { } else {
if o := f.newObjectWithInfo(remote, info); o != nil { o, err := f.newObjectWithInfo(remote, info)
if out.Add(o) { if err != nil {
return true out.SetError(err)
} return true
}
if out.Add(o) {
return true
} }
} }
return false return false
@ -705,7 +709,11 @@ func (o *Object) readMetaData() (err error) {
// } // }
info, _, err := o.fs.readMetaDataForPath(o.srvPath()) info, _, err := o.fs.readMetaDataForPath(o.srvPath())
if err != nil { if err != nil {
fs.Debug(o, "Failed to read info: %s", err) if apiErr, ok := err.(*api.Error); ok {
if apiErr.ErrorInfo.Code == "itemNotFound" {
return fs.ErrorObjectNotFound
}
}
return err return err
} }
o.setMetaData(info) o.setMetaData(info)

5
rclone.go
View File

@ -383,8 +383,9 @@ func NewFsSrc(remote string) fs.Fs {
log.Fatalf("Can't limit to single files when using filters: %v", remote) log.Fatalf("Can't limit to single files when using filters: %v", remote)
} }
// Limit transfers to this file // Limit transfers to this file
fs.Config.Filter.AddFile(path.Base(fsPath)) err = fs.Config.Filter.AddFile(path.Base(fsPath))
} else if err != nil { }
if err != nil {
fs.Stats.Error() fs.Stats.Error()
log.Fatalf("Failed to create file system for %q: %v", remote, err) log.Fatalf("Failed to create file system for %q: %v", remote, err)
} }

34
s3/s3.go
View File

@ -342,8 +342,8 @@ func NewFs(name, root string) (fs.Fs, error) {
// Return an Object from a path // Return an Object from a path
// //
// May return nil if an error occurred //If it can't be found it returns the error ErrorObjectNotFound.
func (f *Fs) newObjectWithInfo(remote string, info *s3.Object) fs.Object { func (f *Fs) newObjectWithInfo(remote string, info *s3.Object) (fs.Object, error) {
o := &Object{ o := &Object{
fs: f, fs: f,
remote: remote, remote: remote,
@ -361,17 +361,15 @@ func (f *Fs) newObjectWithInfo(remote string, info *s3.Object) fs.Object {
} else { } else {
err := o.readMetaData() // reads info and meta, returning an error err := o.readMetaData() // reads info and meta, returning an error
if err != nil { if err != nil {
// logged already FsDebug("Failed to read info: %s", err) return nil, err
return nil
} }
} }
return o return o, nil
} }
// NewObject returns an Object from a path // NewObject finds the Object at remote. If it can't be found
// // it returns the error fs.ErrorObjectNotFound.
// May return nil if an error occurred func (f *Fs) NewObject(remote string) (fs.Object, error) {
func (f *Fs) NewObject(remote string) fs.Object {
return f.newObjectWithInfo(remote, nil) return f.newObjectWithInfo(remote, nil)
} }
@ -482,10 +480,12 @@ func (f *Fs) listFiles(out fs.ListOpts, dir string) {
return fs.ErrorListAborted return fs.ErrorListAborted
} }
} else { } else {
if o := f.newObjectWithInfo(remote, object); o != nil { o, err := f.newObjectWithInfo(remote, object)
if out.Add(o) { if err != nil {
return fs.ErrorListAborted return err
} }
if out.Add(o) {
return fs.ErrorListAborted
} }
} }
return nil return nil
@ -634,7 +634,7 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
return f.NewObject(remote), err return f.NewObject(remote)
} }
// Hashes returns the supported hash sets. // Hashes returns the supported hash sets.
@ -697,7 +697,11 @@ func (o *Object) readMetaData() (err error) {
} }
resp, err := o.fs.c.HeadObject(&req) resp, err := o.fs.c.HeadObject(&req)
if err != nil { if err != nil {
fs.Debug(o, "Failed to read info: %s", err) if awsErr, ok := err.(awserr.RequestFailure); ok {
if awsErr.StatusCode() == http.StatusNotFound {
return fs.ErrorObjectNotFound
}
}
return err return err
} }
var size int64 var size int64

45
swift/swift.go
View File

@ -223,8 +223,8 @@ func NewFs(name, root string) (fs.Fs, error) {
// Return an Object from a path // Return an Object from a path
// //
// May return nil if an error occurred // If it can't be found it returns the error fs.ErrorObjectNotFound.
func (f *Fs) newObjectWithInfo(remote string, info *swift.Object) fs.Object { func (f *Fs) newObjectWithInfo(remote string, info *swift.Object) (fs.Object, error) {
o := &Object{ o := &Object{
fs: f, fs: f,
remote: remote, remote: remote,
@ -241,17 +241,15 @@ func (f *Fs) newObjectWithInfo(remote string, info *swift.Object) fs.Object {
} else { } else {
err := o.readMetaData() // reads info and headers, returning an error err := o.readMetaData() // reads info and headers, returning an error
if err != nil { if err != nil {
fs.Debug(o, "Failed to read metadata: %s", err) return nil, err
return nil
} }
} }
return o return o, nil
} }
// NewObject returns an Object from a path // NewObject finds the Object at remote. If it can't be found it
// // returns the error fs.ErrorObjectNotFound.
// May return nil if an error occurred func (f *Fs) NewObject(remote string) (fs.Object, error) {
func (f *Fs) NewObject(remote string) fs.Object {
return f.newObjectWithInfo(remote, nil) return f.newObjectWithInfo(remote, nil)
} }
@ -331,12 +329,14 @@ func (f *Fs) listFiles(out fs.ListOpts, dir string) {
return fs.ErrorListAborted return fs.ErrorListAborted
} }
} else { } else {
if o := f.newObjectWithInfo(remote, object); o != nil { o, err := f.newObjectWithInfo(remote, object)
// Storable does a full metadata read on 0 size objects which might be dynamic large objects if err != nil {
if o.Storable() { return err
if out.Add(o) { }
return fs.ErrorListAborted // Storable does a full metadata read on 0 size objects which might be dynamic large objects
} if o.Storable() {
if out.Add(o) {
return fs.ErrorListAborted
} }
} }
} }
@ -437,9 +437,11 @@ func (f *Fs) Purge() error {
go func() { go func() {
err = f.list("", fs.MaxLevel, func(remote string, object *swift.Object, isDirectory bool) error { err = f.list("", fs.MaxLevel, func(remote string, object *swift.Object, isDirectory bool) error {
if !isDirectory { if !isDirectory {
if o := f.newObjectWithInfo(remote, object); o != nil { o, err := f.newObjectWithInfo(remote, object)
toBeDeleted <- o if err != nil {
return err
} }
toBeDeleted <- o
} }
return nil return nil
}) })
@ -472,7 +474,7 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
return f.NewObject(remote), nil return f.NewObject(remote)
} }
// Hashes returns the supported hash sets. // Hashes returns the supported hash sets.
@ -525,7 +527,7 @@ func (o *Object) Hash(t fs.HashType) (string, error) {
func (o *Object) hasHeader(header string) (bool, error) { func (o *Object) hasHeader(header string) (bool, error) {
err := o.readMetaData() err := o.readMetaData()
if err != nil { if err != nil {
if err == swift.ObjectNotFound { if err == fs.ErrorObjectNotFound {
return false, nil return false, nil
} }
return false, err return false, err
@ -552,12 +554,17 @@ func (o *Object) Size() int64 {
// readMetaData gets the metadata if it hasn't already been fetched // readMetaData gets the metadata if it hasn't already been fetched
// //
// it also sets the info // it also sets the info
//
// it returns fs.ErrorObjectNotFound if the object isn't found
func (o *Object) readMetaData() (err error) { func (o *Object) readMetaData() (err error) {
if o.headers != nil { if o.headers != nil {
return nil return nil
} }
info, h, err := o.fs.c.Object(o.fs.container, o.fs.root+o.remote) info, h, err := o.fs.c.Object(o.fs.container, o.fs.root+o.remote)
if err != nil { if err != nil {
if err == swift.ObjectNotFound {
return fs.ErrorObjectNotFound
}
return err return err
} }
o.info = info o.info = info

31
yandex/yandex.go
View File

@ -263,10 +263,12 @@ func (f *Fs) List(out fs.ListOpts, dir string) {
return fs.ErrorListAborted return fs.ErrorListAborted
} }
} else { } else {
if o := f.newObjectWithInfo(remote, object); o != nil { o, err := f.newObjectWithInfo(remote, object)
if out.Add(o) { if err != nil {
return fs.ErrorListAborted return err
} }
if out.Add(o) {
return fs.ErrorListAborted
} }
} }
return nil return nil
@ -295,17 +297,16 @@ func (f *Fs) List(out fs.ListOpts, dir string) {
} }
} }
// NewObject returns an Object from a path // NewObject finds the Object at remote. If it can't be found it
// // returns the error fs.ErrorObjectNotFound.
// May return nil if an error occurred func (f *Fs) NewObject(remote string) (fs.Object, error) {
func (f *Fs) NewObject(remote string) fs.Object {
return f.newObjectWithInfo(remote, nil) return f.newObjectWithInfo(remote, nil)
} }
// Return an Object from a path // Return an Object from a path
// //
// May return nil if an error occurred // If it can't be found it returns the error fs.ErrorObjectNotFound.
func (f *Fs) newObjectWithInfo(remote string, info *yandex.ResourceInfoResponse) fs.Object { func (f *Fs) newObjectWithInfo(remote string, info *yandex.ResourceInfoResponse) (fs.Object, error) {
o := &Object{ o := &Object{
fs: f, fs: f,
remote: remote, remote: remote,
@ -315,11 +316,10 @@ func (f *Fs) newObjectWithInfo(remote string, info *yandex.ResourceInfoResponse)
} else { } else {
err := o.readMetaData() err := o.readMetaData()
if err != nil { if err != nil {
fs.Debug(f, "Couldn't get object '%s' metadata: %s", o.remotePath(), err) return nil, err
return nil
} }
} }
return o return o, nil
} }
// setMetaData sets the fs data from a storage.Object // setMetaData sets the fs data from a storage.Object
@ -355,6 +355,11 @@ func (o *Object) readMetaData() (err error) {
var opt2 yandex.ResourceInfoRequestOptions var opt2 yandex.ResourceInfoRequestOptions
ResourceInfoResponse, err := o.fs.yd.NewResourceInfoRequest(o.remotePath(), opt2).Exec() ResourceInfoResponse, err := o.fs.yd.NewResourceInfoRequest(o.remotePath(), opt2).Exec()
if err != nil { if err != nil {
if dcErr, ok := err.(yandex.DiskClientError); ok {
if dcErr.Code == "DiskNotFoundError" {
return fs.ErrorObjectNotFound
}
}
return err return err
} }
o.setMetaData(ResourceInfoResponse) o.setMetaData(ResourceInfoResponse)