From db1995e63a6e8d7c8dc46c689596cc9db941896e Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 2 Aug 2017 16:51:24 +0100 Subject: [PATCH] Add MergeDirs optional interface and implement it for drive --- drive/drive.go | 70 +++++++++++++++++++++++++++++++++++-------- fs/fs.go | 17 +++++++++++ fs/operations_test.go | 34 +++++++++++++++++++++ 3 files changed, 109 insertions(+), 12 deletions(-) diff --git a/drive/drive.go b/drive/drive.go index ff277e7ad..745a60668 100644 --- a/drive/drive.go +++ b/drive/drive.go @@ -723,6 +723,46 @@ func (f *Fs) PutUnchecked(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOpt return o, nil } +// MergeDirs merges the contents of all the directories passed +// in into the first one and rmdirs the other directories. +func (f *Fs) MergeDirs(dirs []fs.Directory) error { + if len(dirs) < 2 { + return nil + } + dstDir := dirs[0] + for _, srcDir := range dirs[1:] { + // list the the objects + infos := []*drive.File{} + _, err := f.list(srcDir.ID(), "", false, false, true, func(info *drive.File) bool { + infos = append(infos, info) + return false + }) + if err != nil { + return errors.Wrapf(err, "MergeDirs list failed on %v", srcDir) + } + // move them into place + for _, info := range infos { + fs.Infof(srcDir, "merging %q", info.Title) + // Move the file into the destination + err = f.pacer.Call(func() (bool, error) { + info.Parents = []*drive.ParentReference{{Id: dstDir.ID()}} + info, err = f.svc.Files.Patch(info.Id, info).SetModifiedDate(true).Fields(googleapi.Field(partialFields)).SupportsTeamDrives(f.isTeamDrive).Do() + return shouldRetry(err) + }) + if err != nil { + return errors.Wrapf(err, "MergDirs move failed on %q in %v", info.Title, srcDir) + } + } + // rmdir (into trash) the now empty source directory + err = f.rmdir(srcDir.ID(), true) + if err != nil { + fs.Infof(srcDir, "removing empty directory") + return errors.Wrapf(err, "MergDirs move failed to rmdir %q", srcDir) + } + } + return nil +} + // Mkdir creates the container if it doesn't exist func (f *Fs) Mkdir(dir string) error { err := f.dirCache.FindRoot(true) @@ -735,6 +775,19 @@ func (f *Fs) Mkdir(dir string) error { return err } +// Rmdir deletes a directory unconditionally by ID +func (f *Fs) rmdir(directoryID string, useTrash bool) error { + return f.pacer.Call(func() (bool, error) { + var err error + if useTrash { + _, err = f.svc.Files.Trash(directoryID).Fields(googleapi.Field(partialFields)).SupportsTeamDrives(f.isTeamDrive).Do() + } else { + err = f.svc.Files.Delete(directoryID).Fields(googleapi.Field(partialFields)).SupportsTeamDrives(f.isTeamDrive).Do() + } + return shouldRetry(err) + }) +} + // Rmdir deletes a directory // // Returns an error if it isn't empty @@ -761,19 +814,11 @@ func (f *Fs) Rmdir(dir string) error { if found { return errors.Errorf("directory not empty") } - // Delete the directory if it isn't the root if root != "" { - err = f.pacer.Call(func() (bool, error) { - // trash the directory if it had trashed files - // in or the user wants to trash, otherwise - // delete it. - if trashedFiles || *driveUseTrash { - _, err = f.svc.Files.Trash(directoryID).Fields(googleapi.Field(partialFields)).SupportsTeamDrives(f.isTeamDrive).Do() - } else { - err = f.svc.Files.Delete(directoryID).Fields(googleapi.Field(partialFields)).SupportsTeamDrives(f.isTeamDrive).Do() - } - return shouldRetry(err) - }) + // trash the directory if it had trashed files + // in or the user wants to trash, otherwise + // delete it. + err = f.rmdir(directoryID, trashedFiles || *driveUseTrash) if err != nil { return err } @@ -1375,6 +1420,7 @@ var ( _ fs.DirCacheFlusher = (*Fs)(nil) _ fs.DirChangeNotifier = (*Fs)(nil) _ fs.PutUncheckeder = (*Fs)(nil) + _ fs.MergeDirser = (*Fs)(nil) _ fs.Object = (*Object)(nil) _ fs.MimeTyper = &Object{} ) diff --git a/fs/fs.go b/fs/fs.go index 38e3ae010..6830ac1d4 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -318,6 +318,10 @@ type Features struct { // nil and the error PutStream func(in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error) + // MergeDirs merges the contents of all the directories passed + // in into the first one and rmdirs the other directories. + MergeDirs func([]Directory) error + // CleanUp the trash in the Fs // // Implement this if you have a way of emptying the trash or @@ -374,6 +378,9 @@ func (ft *Features) Fill(f Fs) *Features { if do, ok := f.(PutStreamer); ok { ft.PutStream = do.PutStream } + if do, ok := f.(MergeDirser); ok { + ft.MergeDirs = do.MergeDirs + } if do, ok := f.(CleanUpper); ok { ft.CleanUp = do.CleanUp } @@ -422,6 +429,9 @@ func (ft *Features) Mask(f Fs) *Features { if mask.PutStream == nil { ft.PutStream = nil } + if mask.MergeDirs == nil { + ft.MergeDirs = nil + } if mask.CleanUp == nil { ft.CleanUp = nil } @@ -538,6 +548,13 @@ type PutStreamer interface { PutStream(in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error) } +// MergeDirser is an option interface for Fs +type MergeDirser interface { + // MergeDirs merges the contents of all the directories passed + // in into the first one and rmdirs the other directories. + MergeDirs([]Directory) error +} + // CleanUpper is an optional interfaces for Fs type CleanUpper interface { // CleanUp the trash in the Fs diff --git a/fs/operations_test.go b/fs/operations_test.go index 1975715ae..d7a75e8e4 100644 --- a/fs/operations_test.go +++ b/fs/operations_test.go @@ -667,6 +667,40 @@ func TestDeduplicateRename(t *testing.T) { })) } +// This should really be a unit test, but the test framework there +// doesn't have enough tools to make it easy +func TestMergeDirs(t *testing.T) { + r := NewRun(t) + defer r.Finalise() + + mergeDirs := r.fremote.Features().MergeDirs + if mergeDirs == nil { + t.Skip("Can't merge directories") + } + + file1 := r.WriteObject("dupe1/one.txt", "This is one", t1) + file2 := r.WriteObject("dupe2/two.txt", "This is one too", t2) + file3 := r.WriteObject("dupe3/three.txt", "This is another one", t3) + + objs, dirs, err := fs.WalkGetAll(r.fremote, "", true, 1) + require.NoError(t, err) + assert.Equal(t, 3, len(dirs)) + assert.Equal(t, 0, len(objs)) + + err = mergeDirs(dirs) + require.NoError(t, err) + + file2.Path = "dupe1/two.txt" + file3.Path = "dupe1/three.txt" + fstest.CheckItems(t, r.fremote, file1, file2, file3) + + objs, dirs, err = fs.WalkGetAll(r.fremote, "", true, 1) + require.NoError(t, err) + assert.Equal(t, 1, len(dirs)) + assert.Equal(t, 0, len(objs)) + assert.Equal(t, "dupe1", dirs[0].Remote()) +} + func TestCat(t *testing.T) { r := NewRun(t) defer r.Finalise()