diff --git a/fs/cache/cache.go b/fs/cache/cache.go index 106c0bbc5..dcc94adff 100644 --- a/fs/cache/cache.go +++ b/fs/cache/cache.go @@ -104,6 +104,19 @@ func Get(ctx context.Context, fsString string) (f fs.Fs, err error) { return GetFn(ctx, fsString, fs.NewFs) } +// GetArr gets []fs.Fs from []fsStrings either from the cache or creates it afresh +func GetArr(ctx context.Context, fsStrings []string) (f []fs.Fs, err error) { + var fArr []fs.Fs + for _, fsString := range fsStrings { + f1, err1 := GetFn(ctx, fsString, fs.NewFs) + if err1 != nil { + return fArr, err1 + } + fArr = append(fArr, f1) + } + return fArr, nil +} + // Put puts an fs.Fs named fsString into the cache func Put(fsString string, f fs.Fs) { canonicalName := fs.ConfigString(f) diff --git a/fs/config.go b/fs/config.go index 8481bd4e0..fa7198e3b 100644 --- a/fs/config.go +++ b/fs/config.go @@ -76,8 +76,8 @@ type ConfigInfo struct { NoUnicodeNormalization bool NoUpdateModTime bool DataRateUnit string - CompareDest string - CopyDest string + CompareDest []string + CopyDest []string BackupDir string Suffix string SuffixKeepExtension bool diff --git a/fs/config/configflags/configflags.go b/fs/config/configflags/configflags.go index 0a4f177c7..b41c380e8 100644 --- a/fs/config/configflags/configflags.go +++ b/fs/config/configflags/configflags.go @@ -81,8 +81,8 @@ func AddFlags(ci *fs.ConfigInfo, flagSet *pflag.FlagSet) { flags.BoolVarP(flagSet, &ci.NoCheckDest, "no-check-dest", "", ci.NoCheckDest, "Don't check the destination, copy regardless.") flags.BoolVarP(flagSet, &ci.NoUnicodeNormalization, "no-unicode-normalization", "", ci.NoUnicodeNormalization, "Don't normalize unicode characters in filenames.") flags.BoolVarP(flagSet, &ci.NoUpdateModTime, "no-update-modtime", "", ci.NoUpdateModTime, "Don't update destination mod-time if files identical.") - flags.StringVarP(flagSet, &ci.CompareDest, "compare-dest", "", ci.CompareDest, "Include additional server-side path during comparison.") - flags.StringVarP(flagSet, &ci.CopyDest, "copy-dest", "", ci.CopyDest, "Implies --compare-dest but also copies files from path into destination.") + flags.StringArrayVarP(flagSet, &ci.CompareDest, "compare-dest", "", nil, "Include additional comma separated server-side paths during comparison.") + flags.StringArrayVarP(flagSet, &ci.CopyDest, "copy-dest", "", nil, "Implies --compare-dest but also copies files from paths into destination.") flags.StringVarP(flagSet, &ci.BackupDir, "backup-dir", "", ci.BackupDir, "Make backups into hierarchy based in DIR.") flags.StringVarP(flagSet, &ci.Suffix, "suffix", "", ci.Suffix, "Suffix to add to changed files.") flags.BoolVarP(flagSet, &ci.SuffixKeepExtension, "suffix-keep-extension", "", ci.SuffixKeepExtension, "Preserve the extension when using --suffix.") @@ -217,7 +217,7 @@ func SetFlags(ci *fs.ConfigInfo) { ci.DeleteMode = fs.DeleteModeDefault } - if ci.CompareDest != "" && ci.CopyDest != "" { + if len(ci.CompareDest) > 0 && len(ci.CopyDest) > 0 { log.Fatalf(`Can't use --compare-dest with --copy-dest.`) } diff --git a/fs/operations/operations.go b/fs/operations/operations.go index 1b3cd69af..6e198c4bf 100644 --- a/fs/operations/operations.go +++ b/fs/operations/operations.go @@ -748,6 +748,16 @@ func SameConfig(fdst, fsrc fs.Info) bool { return fdst.Name() == fsrc.Name() } +// SameConfigArr returns true if any of []fsrcs has same config file entry with fdst +func SameConfigArr(fdst fs.Info, fsrcs []fs.Fs) bool { + for _, fsrc := range fsrcs { + if fdst.Name() == fsrc.Name() { + return true + } + } + return false +} + // Same returns true if fdst and fsrc point to the same underlying Fs func Same(fdst, fsrc fs.Info) bool { return SameConfig(fdst, fsrc) && strings.Trim(fdst.Root(), "/") == strings.Trim(fsrc.Root(), "/") @@ -1354,9 +1364,9 @@ func Rmdirs(ctx context.Context, f fs.Fs, dir string, leaveRoot bool) error { } // GetCompareDest sets up --compare-dest -func GetCompareDest(ctx context.Context) (CompareDest fs.Fs, err error) { +func GetCompareDest(ctx context.Context) (CompareDest []fs.Fs, err error) { ci := fs.GetConfig(ctx) - CompareDest, err = cache.Get(ctx, ci.CompareDest) + CompareDest, err = cache.GetArr(ctx, ci.CompareDest) if err != nil { return nil, fserrors.FatalError(errors.Errorf("Failed to make fs for --compare-dest %q: %v", ci.CompareDest, err)) } @@ -1391,18 +1401,21 @@ func compareDest(ctx context.Context, dst, src fs.Object, CompareDest fs.Fs) (No } // GetCopyDest sets up --copy-dest -func GetCopyDest(ctx context.Context, fdst fs.Fs) (CopyDest fs.Fs, err error) { +func GetCopyDest(ctx context.Context, fdst fs.Fs) (CopyDest []fs.Fs, err error) { ci := fs.GetConfig(ctx) - CopyDest, err = cache.Get(ctx, ci.CopyDest) + CopyDest, err = cache.GetArr(ctx, ci.CopyDest) if err != nil { return nil, fserrors.FatalError(errors.Errorf("Failed to make fs for --copy-dest %q: %v", ci.CopyDest, err)) } - if !SameConfig(fdst, CopyDest) { + if !SameConfigArr(fdst, CopyDest) { return nil, fserrors.FatalError(errors.New("parameter to --copy-dest has to be on the same remote as destination")) } - if CopyDest.Features().Copy == nil { - return nil, fserrors.FatalError(errors.New("can't use --copy-dest on a remote which doesn't support server-side copy")) + for _, cf := range CopyDest { + if cf.Features().Copy == nil { + return nil, fserrors.FatalError(errors.New("can't use --copy-dest on a remote which doesn't support server side copy")) + } } + return CopyDest, nil } @@ -1457,12 +1470,22 @@ func copyDest(ctx context.Context, fdst fs.Fs, dst, src fs.Object, CopyDest, bac // does not need to be copied // // Returns True if src does not need to be copied -func CompareOrCopyDest(ctx context.Context, fdst fs.Fs, dst, src fs.Object, CompareOrCopyDest, backupDir fs.Fs) (NoNeedTransfer bool, err error) { +func CompareOrCopyDest(ctx context.Context, fdst fs.Fs, dst, src fs.Object, CompareOrCopyDest []fs.Fs, backupDir fs.Fs) (NoNeedTransfer bool, err error) { ci := fs.GetConfig(ctx) - if ci.CompareDest != "" { - return compareDest(ctx, dst, src, CompareOrCopyDest) - } else if ci.CopyDest != "" { - return copyDest(ctx, fdst, dst, src, CompareOrCopyDest, backupDir) + if len(ci.CompareDest) > 0 { + for _, compareF := range CompareOrCopyDest { + NoNeedTransfer, err := compareDest(ctx, dst, src, compareF) + if NoNeedTransfer || err != nil { + return NoNeedTransfer, err + } + } + } else if len(ci.CopyDest) > 0 { + for _, copyF := range CompareOrCopyDest { + NoNeedTransfer, err := copyDest(ctx, fdst, dst, src, copyF, backupDir) + if NoNeedTransfer || err != nil { + return NoNeedTransfer, err + } + } } return false, nil } @@ -1732,19 +1755,20 @@ func moveOrCopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName str return err } - var backupDir, copyDestDir fs.Fs + var backupDir fs.Fs + var copyDestDir []fs.Fs if ci.BackupDir != "" || ci.Suffix != "" { backupDir, err = BackupDir(ctx, fdst, fsrc, srcFileName) if err != nil { return errors.Wrap(err, "creating Fs for --backup-dir failed") } } - if ci.CompareDest != "" { + if len(ci.CompareDest) > 0 { copyDestDir, err = GetCompareDest(ctx) if err != nil { return err } - } else if ci.CopyDest != "" { + } else if len(ci.CopyDest) > 0 { copyDestDir, err = GetCopyDest(ctx, fdst) if err != nil { return err diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go index 796b49ed4..2421b5b9b 100644 --- a/fs/operations/operations_test.go +++ b/fs/operations/operations_test.go @@ -909,9 +909,9 @@ func TestCopyFileCompareDest(t *testing.T) { r := fstest.NewRun(t) defer r.Finalise() - ci.CompareDest = r.FremoteName + "/CompareDest" + ci.CompareDest = []string{r.FremoteName + "/CompareDest"} defer func() { - ci.CompareDest = "" + ci.CompareDest = nil }() fdst, err := fs.NewFs(ctx, r.FremoteName+"/dst") require.NoError(t, err) @@ -995,9 +995,9 @@ func TestCopyFileCopyDest(t *testing.T) { t.Skip("Skipping test as remote does not support server-side copy") } - ci.CopyDest = r.FremoteName + "/CopyDest" + ci.CopyDest = []string{r.FremoteName + "/CopyDest"} defer func() { - ci.CopyDest = "" + ci.CopyDest = nil }() fdst, err := fs.NewFs(ctx, r.FremoteName+"/dst") diff --git a/fs/sync/sync.go b/fs/sync/sync.go index 59cb7233e..e8ce10ad1 100644 --- a/fs/sync/sync.go +++ b/fs/sync/sync.go @@ -70,7 +70,7 @@ type syncCopyMove struct { trackRenamesWg sync.WaitGroup // wg for background track renames trackRenamesCh chan fs.Object // objects are pumped in here renameCheck []fs.Object // accumulate files to check for rename here - compareCopyDest fs.Fs // place to check for files to server-side copy + compareCopyDest []fs.Fs // place to check for files to server side copy backupDir fs.Fs // place to store overwrites/deletes checkFirst bool // if set run all the checkers before starting transfers } @@ -212,13 +212,13 @@ func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete return nil, err } } - if ci.CompareDest != "" { + if len(ci.CompareDest) > 0 { var err error s.compareCopyDest, err = operations.GetCompareDest(ctx) if err != nil { return nil, err } - } else if ci.CopyDest != "" { + } else if len(ci.CopyDest) > 0 { var err error s.compareCopyDest, err = operations.GetCopyDest(ctx, fdst) if err != nil { @@ -890,7 +890,7 @@ func (s *syncCopyMove) run() error { // Delete empty fsrc subdirectories // if DoMove and --delete-empty-src-dirs flag is set if s.DoMove && s.deleteEmptySrcDirs { - //delete empty subdirectories that were part of the move + // delete empty subdirectories that were part of the move s.processError(s.deleteEmptyDirectories(s.ctx, s.fsrc, s.srcEmptyDirs)) } diff --git a/fs/sync/sync_test.go b/fs/sync/sync_test.go index 37fb2abdf..3d7b54831 100644 --- a/fs/sync/sync_test.go +++ b/fs/sync/sync_test.go @@ -1480,9 +1480,9 @@ func TestSyncCompareDest(t *testing.T) { r := fstest.NewRun(t) defer r.Finalise() - ci.CompareDest = r.FremoteName + "/CompareDest" + ci.CompareDest = []string{r.FremoteName + "/CompareDest"} defer func() { - ci.CompareDest = "" + ci.CompareDest = []string{} }() fdst, err := fs.NewFs(ctx, r.FremoteName+"/dst") @@ -1562,6 +1562,40 @@ func TestSyncCompareDest(t *testing.T) { fstest.CheckItems(t, r.Fremote, file2, file3, file4, file5bdst) } +// Test with multiple CompareDest +func TestSyncMultipleCompareDest(t *testing.T) { + ctx := context.Background() + ci := fs.GetConfig(ctx) + r := fstest.NewRun(t) + defer r.Finalise() + + ci.CompareDest = []string{r.FremoteName + "/pre-dest1", r.FremoteName + "/pre-dest2"} + defer func() { + ci.CompareDest = []string{} + }() + + // check empty dest, new compare + fsrc1 := r.WriteFile("1", "1", t1) + fsrc2 := r.WriteFile("2", "2", t1) + fsrc3 := r.WriteFile("3", "3", t1) + fstest.CheckItems(t, r.Flocal, fsrc1, fsrc2, fsrc3) + + fdest1 := r.WriteObject(ctx, "pre-dest1/1", "1", t1) + fdest2 := r.WriteObject(ctx, "pre-dest2/2", "2", t1) + fstest.CheckItems(t, r.Fremote, fdest1, fdest2) + + accounting.GlobalStats().ResetCounters() + fdst, err := fs.NewFs(ctx, r.FremoteName+"/dest") + require.NoError(t, err) + require.NoError(t, Sync(ctx, fdst, r.Flocal, false)) + + fdest3 := fsrc3 + fdest3.Path = "dest/3" + + fstest.CheckItems(t, fdst, fsrc3) + fstest.CheckItems(t, r.Fremote, fdest1, fdest2, fdest3) +} + // Test with CopyDest set func TestSyncCopyDest(t *testing.T) { ctx := context.Background() @@ -1573,9 +1607,9 @@ func TestSyncCopyDest(t *testing.T) { t.Skip("Skipping test as remote does not support server-side copy") } - ci.CopyDest = r.FremoteName + "/CopyDest" + ci.CopyDest = []string{r.FremoteName + "/CopyDest"} defer func() { - ci.CopyDest = "" + ci.CopyDest = []string{} }() fdst, err := fs.NewFs(ctx, r.FremoteName+"/dst")