From e2717a031e1bfd63db7288ae1c46114b2e5886d4 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Mon, 31 Aug 2015 21:05:51 +0100 Subject: [PATCH] Implement Mover and DirMover interfaces fixes #115 * unit tests * local * drive * dropbox --- drive/drive.go | 87 ++++++++++++++- drive/drive_test.go | 2 + dropbox/dropbox.go | 69 +++++++++++- dropbox/dropbox_test.go | 2 + fstest/fstests/fstests.go | 88 +++++++++++++++ googlecloudstorage/googlecloudstorage_test.go | 2 + local/local.go | 100 ++++++++++++++++-- local/local_test.go | 2 + s3/s3_test.go | 2 + swift/swift_test.go | 2 + 10 files changed, 339 insertions(+), 17 deletions(-) diff --git a/drive/drive.go b/drive/drive.go index 37c099adc..7c01b681e 100644 --- a/drive/drive.go +++ b/drive/drive.go @@ -919,6 +919,79 @@ func (f *FsDrive) Purge() error { return nil } +// Move src to this remote using server side move operations. +// +// This is stored with the remote path given +// +// It returns the destination Object and a possible error +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantMove +func (dstFs *FsDrive) Move(src fs.Object, remote string) (fs.Object, error) { + srcObj, ok := src.(*FsObjectDrive) + if !ok { + fs.Debug(src, "Can't move - not same remote type") + return nil, fs.ErrorCantMove + } + + // Temporary FsObject under construction + dstObj, dstInfo, err := dstFs.createFileInfo(remote, srcObj.ModTime(), srcObj.bytes) + if err != nil { + return nil, err + } + + // Do the move + info, err := dstFs.svc.Files.Patch(srcObj.id, dstInfo).SetModifiedDate(true).Do() + if err != nil { + return nil, err + } + + dstObj.setMetaData(info) + return dstObj, nil +} + +// Move src to this remote using server side move operations. +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantDirMove +// +// If destination exists then return fs.ErrorDirExists +func (dstFs *FsDrive) DirMove(src fs.Fs) error { + srcFs, ok := src.(*FsDrive) + if !ok { + fs.Debug(srcFs, "Can't move directory - not same remote type") + return fs.ErrorCantDirMove + } + + // Check if destination exists + dstFs.resetRoot() + err := dstFs.findRoot(false) + if err == nil { + return fs.ErrorDirExists + } + + // Find ID of parent + directory, leaf := splitPath(dstFs.root) + directoryId, err := dstFs.findDir(directory, true) + if err != nil { + return fmt.Errorf("Couldn't find or make destination directory: %v", err) + } + + // Do the move + patch := drive.File{ + Title: leaf, + Parents: []*drive.ParentReference{{Id: directoryId}}, + } + _, err = dstFs.svc.Files.Patch(srcFs.rootId, &patch).Do() + if err != nil { + return err + } + srcFs.resetRoot() + return nil +} + // ------------------------------------------------------------ // Return the parent Fs @@ -1007,7 +1080,7 @@ func (o *FsObjectDrive) ModTime() time.Time { return modTime } -// Sets the modification time of the local fs object +// Sets the modification time of the drive fs object func (o *FsObjectDrive) SetModTime(modTime time.Time) { err := o.readMetaData() if err != nil { @@ -1111,7 +1184,11 @@ func (o *FsObjectDrive) Remove() error { } // Check the interfaces are satisfied -var _ fs.Fs = &FsDrive{} -var _ fs.Purger = &FsDrive{} -var _ fs.Copier = &FsDrive{} -var _ fs.Object = &FsObjectDrive{} +var ( + _ fs.Fs = (*FsDrive)(nil) + _ fs.Purger = (*FsDrive)(nil) + _ fs.Copier = (*FsDrive)(nil) + _ fs.Mover = (*FsDrive)(nil) + _ fs.DirMover = (*FsDrive)(nil) + _ fs.Object = (*FsObjectDrive)(nil) +) diff --git a/drive/drive_test.go b/drive/drive_test.go index 5882812c3..5c5e5e484 100644 --- a/drive/drive_test.go +++ b/drive/drive_test.go @@ -35,6 +35,8 @@ func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) } func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } func TestFsCopy(t *testing.T) { fstests.TestFsCopy(t) } +func TestFsMove(t *testing.T) { fstests.TestFsMove(t) } +func TestFsDirMove(t *testing.T) { fstests.TestFsDirMove(t) } func TestFsRmdirFull(t *testing.T) { fstests.TestFsRmdirFull(t) } func TestFsPrecision(t *testing.T) { fstests.TestFsPrecision(t) } func TestObjectString(t *testing.T) { fstests.TestObjectString(t) } diff --git a/dropbox/dropbox.go b/dropbox/dropbox.go index 083fc9df2..c97ee451a 100644 --- a/dropbox/dropbox.go +++ b/dropbox/dropbox.go @@ -452,6 +452,63 @@ func (f *FsDropbox) Purge() error { return err } +// Move src to this remote using server side move operations. +// +// This is stored with the remote path given +// +// It returns the destination Object and a possible error +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantMove +func (dstFs *FsDropbox) Move(src fs.Object, remote string) (fs.Object, error) { + srcObj, ok := src.(*FsObjectDropbox) + if !ok { + fs.Debug(src, "Can't move - not same remote type") + return nil, fs.ErrorCantMove + } + + // Temporary FsObject under construction + dstObj := &FsObjectDropbox{dropbox: dstFs, remote: remote} + + srcPath := srcObj.remotePath() + dstPath := dstObj.remotePath() + entry, err := dstFs.db.Move(srcPath, dstPath) + if err != nil { + return nil, fmt.Errorf("Move failed: %s", err) + } + dstObj.setMetadataFromEntry(entry) + return dstObj, nil +} + +// Move src to this remote using server side move operations. +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantDirMove +// +// If destination exists then return fs.ErrorDirExists +func (dstFs *FsDropbox) DirMove(src fs.Fs) error { + srcFs, ok := src.(*FsDropbox) + if !ok { + fs.Debug(srcFs, "Can't move directory - not same remote type") + return fs.ErrorCantDirMove + } + + // Check if destination exists + entry, err := dstFs.db.Metadata(dstFs.slashRoot, false, false, "", "", metadataLimit) + if err == nil && !entry.IsDeleted { + return fs.ErrorDirExists + } + + // Do the move + _, err = dstFs.db.Move(srcFs.slashRoot, dstFs.slashRoot) + if err != nil { + return fmt.Errorf("MoveDir failed: %v", err) + } + return nil +} + // ------------------------------------------------------------ // Return the parent Fs @@ -601,7 +658,11 @@ func (o *FsObjectDropbox) Remove() error { } // Check the interfaces are satisfied -var _ fs.Fs = &FsDropbox{} -var _ fs.Copier = &FsDropbox{} -var _ fs.Purger = &FsDropbox{} -var _ fs.Object = &FsObjectDropbox{} +var ( + _ fs.Fs = (*FsDropbox)(nil) + _ fs.Copier = (*FsDropbox)(nil) + _ fs.Purger = (*FsDropbox)(nil) + _ fs.Mover = (*FsDropbox)(nil) + _ fs.DirMover = (*FsDropbox)(nil) + _ fs.Object = (*FsObjectDropbox)(nil) +) diff --git a/dropbox/dropbox_test.go b/dropbox/dropbox_test.go index 3de713223..677c92ee8 100644 --- a/dropbox/dropbox_test.go +++ b/dropbox/dropbox_test.go @@ -35,6 +35,8 @@ func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) } func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } func TestFsCopy(t *testing.T) { fstests.TestFsCopy(t) } +func TestFsMove(t *testing.T) { fstests.TestFsMove(t) } +func TestFsDirMove(t *testing.T) { fstests.TestFsDirMove(t) } func TestFsRmdirFull(t *testing.T) { fstests.TestFsRmdirFull(t) } func TestFsPrecision(t *testing.T) { fstests.TestFsPrecision(t) } func TestObjectString(t *testing.T) { fstests.TestObjectString(t) } diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go index 0d0d05283..35a2a3e9e 100644 --- a/fstest/fstests/fstests.go +++ b/fstest/fstests/fstests.go @@ -286,6 +286,94 @@ func TestFsCopy(t *testing.T) { } +func TestFsMove(t *testing.T) { + skipIfNotOk(t) + + // Check have Move + _, ok := remote.(fs.Mover) + if !ok { + t.Skip("FS has no Mover interface") + } + + var file1Move = file1 + file1Move.Path += "-move" + + // do the move + src := findObject(t, file1.Path) + dst, err := remote.(fs.Mover).Move(src, file1Move.Path) + if err != nil { + t.Fatalf("Move failed: %v", err) + } + + // check file exists in new listing + fstest.CheckListing(t, remote, []fstest.Item{file2, file1Move}) + + // Check dst lightly - list above has checked ModTime/Md5sum + if dst.Remote() != file1Move.Path { + t.Errorf("object path: want %q got %q", file1Move.Path, dst.Remote()) + } + + // move it back + src = findObject(t, file1Move.Path) + _, err = remote.(fs.Mover).Move(src, file1.Path) + if err != nil { + t.Errorf("Move failed: %v", err) + } + + // check file exists in new listing + fstest.CheckListing(t, remote, []fstest.Item{file2, file1}) +} + +// Move src to this remote using server side move operations. +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantDirMove +// +// If destination exists then return fs.ErrorDirExists +func TestFsDirMove(t *testing.T) { + skipIfNotOk(t) + + // Check have DirMove + _, ok := remote.(fs.DirMover) + if !ok { + t.Skip("FS has no DirMover interface") + } + + // Check it can't move onto itself + err := remote.(fs.DirMover).DirMove(remote) + if err != fs.ErrorDirExists { + t.Errorf("Expecting fs.ErrorDirExists got: %v", err) + } + + // new remote + newRemote, removeNewRemote, err := fstest.RandomRemote(RemoteName, false) + if err != nil { + t.Fatalf("Failed to create remote: %v", err) + } + defer removeNewRemote() + + // try the move + err = newRemote.(fs.DirMover).DirMove(remote) + if err != nil { + t.Errorf("Failed to DirMove: %v", err) + } + + // check remotes + fstest.CheckListing(t, remote, []fstest.Item{}) + fstest.CheckListing(t, newRemote, []fstest.Item{file2, file1}) + + // move it back + err = remote.(fs.DirMover).DirMove(newRemote) + if err != nil { + t.Errorf("Failed to DirMove: %v", err) + } + + // check remotes + fstest.CheckListing(t, remote, []fstest.Item{file2, file1}) + fstest.CheckListing(t, newRemote, []fstest.Item{}) +} + func TestFsRmdirFull(t *testing.T) { skipIfNotOk(t) err := remote.Rmdir() diff --git a/googlecloudstorage/googlecloudstorage_test.go b/googlecloudstorage/googlecloudstorage_test.go index fd7e24ba8..6fb3a29c2 100644 --- a/googlecloudstorage/googlecloudstorage_test.go +++ b/googlecloudstorage/googlecloudstorage_test.go @@ -35,6 +35,8 @@ func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) } func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } func TestFsCopy(t *testing.T) { fstests.TestFsCopy(t) } +func TestFsMove(t *testing.T) { fstests.TestFsMove(t) } +func TestFsDirMove(t *testing.T) { fstests.TestFsDirMove(t) } func TestFsRmdirFull(t *testing.T) { fstests.TestFsRmdirFull(t) } func TestFsPrecision(t *testing.T) { fstests.TestFsPrecision(t) } func TestObjectString(t *testing.T) { fstests.TestObjectString(t) } diff --git a/local/local.go b/local/local.go index f6dae8b99..893195041 100644 --- a/local/local.go +++ b/local/local.go @@ -87,19 +87,24 @@ func (f *FsLocal) String() string { return fmt.Sprintf("Local file system at %s", f.root) } +// newFsObject makes a half completed FsObjectLocal +func (f *FsLocal) newFsObject(remote string) *FsObjectLocal { + remote = filepath.ToSlash(remote) + dstPath := path.Join(f.root, remote) + return &FsObjectLocal{local: f, remote: remote, path: dstPath} +} + // Return an FsObject from a path // // May return nil if an error occurred func (f *FsLocal) newFsObjectWithInfo(remote string, info os.FileInfo) fs.Object { - remote = filepath.ToSlash(remote) - path := path.Join(f.root, remote) - o := &FsObjectLocal{local: f, remote: remote, path: path} + o := f.newFsObject(remote) if info != nil { o.info = info } else { err := o.lstat() if err != nil { - fs.Debug(o, "Failed to stat %s: %s", path, err) + fs.Debug(o, "Failed to stat %s: %s", o.path, err) return nil } } @@ -210,9 +215,8 @@ func (f *FsLocal) ListDir() fs.DirChan { // Puts the FsObject to the local filesystem func (f *FsLocal) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) { - dstPath := path.Join(f.root, remote) // Temporary FsObject under construction - info filled in by Update() - o := &FsObjectLocal{local: f, remote: remote, path: dstPath} + o := f.newFsObject(remote) err := o.Update(in, modTime, size) if err != nil { return nil, err @@ -308,6 +312,79 @@ func (f *FsLocal) Purge() error { return os.RemoveAll(f.root) } +// Move src to this remote using server side move operations. +// +// This is stored with the remote path given +// +// It returns the destination Object and a possible error +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantMove +func (dstFs *FsLocal) Move(src fs.Object, remote string) (fs.Object, error) { + srcObj, ok := src.(*FsObjectLocal) + if !ok { + fs.Debug(src, "Can't move - not same remote type") + return nil, fs.ErrorCantMove + } + + // Temporary FsObject under construction + dstObj := dstFs.newFsObject(remote) + + // Check it is a file if it exists + err := dstObj.lstat() + if os.IsNotExist(err) { + // OK + } else if err != nil { + return nil, err + } else if !dstObj.info.Mode().IsRegular() { + // It isn't a file + return nil, fmt.Errorf("Can't move file onto non-file") + } + + // Create destination + err = dstObj.mkdirAll() + if err != nil { + return nil, err + } + + // Do the move + err = os.Rename(srcObj.path, dstObj.path) + if err != nil { + return nil, err + } + + // Update the info + err = dstObj.lstat() + if err != nil { + return nil, err + } + + return dstObj, nil +} + +// Move src to this remote using server side move operations. +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantDirMove +// +// If destination exists then return fs.ErrorDirExists +func (dstFs *FsLocal) DirMove(src fs.Fs) error { + srcFs, ok := src.(*FsLocal) + if !ok { + fs.Debug(srcFs, "Can't move directory - not same remote type") + return fs.ErrorCantDirMove + } + // Check if destination exists + _, err := os.Lstat(dstFs.root) + if !os.IsNotExist(err) { + return fs.ErrorDirExists + } + // Do the move + return os.Rename(srcFs.root, dstFs.root) +} + // ------------------------------------------------------------ // Return the parent Fs @@ -438,10 +515,15 @@ func (o *FsObjectLocal) Open() (in io.ReadCloser, err error) { return } +// mkdirAll makes all the directories needed to store the object +func (o *FsObjectLocal) mkdirAll() error { + dir := path.Dir(o.path) + return os.MkdirAll(dir, 0777) +} + // Update the object from in with modTime and size func (o *FsObjectLocal) Update(in io.Reader, modTime time.Time, size int64) error { - dir := path.Dir(o.path) - err := os.MkdirAll(dir, 0777) + err := o.mkdirAll() if err != nil { return err } @@ -489,4 +571,6 @@ func (o *FsObjectLocal) Remove() error { // Check the interfaces are satisfied var _ fs.Fs = &FsLocal{} var _ fs.Purger = &FsLocal{} +var _ fs.Mover = &FsLocal{} +var _ fs.DirMover = &FsLocal{} var _ fs.Object = &FsObjectLocal{} diff --git a/local/local_test.go b/local/local_test.go index b16a5d93a..82c98a3e1 100644 --- a/local/local_test.go +++ b/local/local_test.go @@ -35,6 +35,8 @@ func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) } func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } func TestFsCopy(t *testing.T) { fstests.TestFsCopy(t) } +func TestFsMove(t *testing.T) { fstests.TestFsMove(t) } +func TestFsDirMove(t *testing.T) { fstests.TestFsDirMove(t) } func TestFsRmdirFull(t *testing.T) { fstests.TestFsRmdirFull(t) } func TestFsPrecision(t *testing.T) { fstests.TestFsPrecision(t) } func TestObjectString(t *testing.T) { fstests.TestObjectString(t) } diff --git a/s3/s3_test.go b/s3/s3_test.go index 5d490f83b..4d928ec40 100644 --- a/s3/s3_test.go +++ b/s3/s3_test.go @@ -35,6 +35,8 @@ func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) } func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } func TestFsCopy(t *testing.T) { fstests.TestFsCopy(t) } +func TestFsMove(t *testing.T) { fstests.TestFsMove(t) } +func TestFsDirMove(t *testing.T) { fstests.TestFsDirMove(t) } func TestFsRmdirFull(t *testing.T) { fstests.TestFsRmdirFull(t) } func TestFsPrecision(t *testing.T) { fstests.TestFsPrecision(t) } func TestObjectString(t *testing.T) { fstests.TestObjectString(t) } diff --git a/swift/swift_test.go b/swift/swift_test.go index 569d8c753..739b86c24 100644 --- a/swift/swift_test.go +++ b/swift/swift_test.go @@ -35,6 +35,8 @@ func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) } func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } func TestFsCopy(t *testing.T) { fstests.TestFsCopy(t) } +func TestFsMove(t *testing.T) { fstests.TestFsMove(t) } +func TestFsDirMove(t *testing.T) { fstests.TestFsDirMove(t) } func TestFsRmdirFull(t *testing.T) { fstests.TestFsRmdirFull(t) } func TestFsPrecision(t *testing.T) { fstests.TestFsPrecision(t) } func TestObjectString(t *testing.T) { fstests.TestObjectString(t) }