community
/
rclone
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: add multiple paths support to `--compare-dest` and `--copy-dest` flag

This commit is contained in:
K265 2020-11-16 11:04:29 +08:00 committed by Nick Craig-Wood
parent 23b12c39bd
commit 930bca2478
7 changed files with 103 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
fs/cache/cache.go vendored
View File

@ -104,6 +104,19 @@ func Get(ctx context.Context, fsString string) (f fs.Fs, err error) {
return GetFn(ctx, fsString, fs.NewFs)
}
// GetArr gets []fs.Fs from []fsStrings either from the cache or creates it afresh
func GetArr(ctx context.Context, fsStrings []string) (f []fs.Fs, err error) {
var fArr []fs.Fs
for _, fsString := range fsStrings {
f1, err1 := GetFn(ctx, fsString, fs.NewFs)
if err1 != nil {
return fArr, err1
}
fArr = append(fArr, f1)
}
return fArr, nil
}
// Put puts an fs.Fs named fsString into the cache
func Put(fsString string, f fs.Fs) {
canonicalName := fs.ConfigString(f)

4
fs/config.go
View File

@ -76,8 +76,8 @@ type ConfigInfo struct {
NoUnicodeNormalization bool
NoUpdateModTime bool
DataRateUnit string
CompareDest string
CopyDest string
CompareDest []string
CopyDest []string
BackupDir string
Suffix string
SuffixKeepExtension bool

6
fs/config/configflags/configflags.go
View File

@ -81,8 +81,8 @@ func AddFlags(ci *fs.ConfigInfo, flagSet *pflag.FlagSet) {
flags.BoolVarP(flagSet, &ci.NoCheckDest, "no-check-dest", "", ci.NoCheckDest, "Don't check the destination, copy regardless.")
flags.BoolVarP(flagSet, &ci.NoUnicodeNormalization, "no-unicode-normalization", "", ci.NoUnicodeNormalization, "Don't normalize unicode characters in filenames.")
flags.BoolVarP(flagSet, &ci.NoUpdateModTime, "no-update-modtime", "", ci.NoUpdateModTime, "Don't update destination mod-time if files identical.")
flags.StringVarP(flagSet, &ci.CompareDest, "compare-dest", "", ci.CompareDest, "Include additional server-side path during comparison.")
flags.StringVarP(flagSet, &ci.CopyDest, "copy-dest", "", ci.CopyDest, "Implies --compare-dest but also copies files from path into destination.")
flags.StringArrayVarP(flagSet, &ci.CompareDest, "compare-dest", "", nil, "Include additional comma separated server-side paths during comparison.")
flags.StringArrayVarP(flagSet, &ci.CopyDest, "copy-dest", "", nil, "Implies --compare-dest but also copies files from paths into destination.")
flags.StringVarP(flagSet, &ci.BackupDir, "backup-dir", "", ci.BackupDir, "Make backups into hierarchy based in DIR.")
flags.StringVarP(flagSet, &ci.Suffix, "suffix", "", ci.Suffix, "Suffix to add to changed files.")
flags.BoolVarP(flagSet, &ci.SuffixKeepExtension, "suffix-keep-extension", "", ci.SuffixKeepExtension, "Preserve the extension when using --suffix.")
@ -217,7 +217,7 @@ func SetFlags(ci *fs.ConfigInfo) {
ci.DeleteMode = fs.DeleteModeDefault
}
if ci.CompareDest != "" && ci.CopyDest != "" {
if len(ci.CompareDest) > 0 && len(ci.CopyDest) > 0 {
log.Fatalf(`Can't use --compare-dest with --copy-dest.`)
}

54
fs/operations/operations.go
View File

@ -748,6 +748,16 @@ func SameConfig(fdst, fsrc fs.Info) bool {
return fdst.Name() == fsrc.Name()
}
// SameConfigArr returns true if any of []fsrcs has same config file entry with fdst
func SameConfigArr(fdst fs.Info, fsrcs []fs.Fs) bool {
for _, fsrc := range fsrcs {
if fdst.Name() == fsrc.Name() {
return true
}
}
return false
}
// Same returns true if fdst and fsrc point to the same underlying Fs
func Same(fdst, fsrc fs.Info) bool {
return SameConfig(fdst, fsrc) && strings.Trim(fdst.Root(), "/") == strings.Trim(fsrc.Root(), "/")
@ -1354,9 +1364,9 @@ func Rmdirs(ctx context.Context, f fs.Fs, dir string, leaveRoot bool) error {
}
// GetCompareDest sets up --compare-dest
func GetCompareDest(ctx context.Context) (CompareDest fs.Fs, err error) {
func GetCompareDest(ctx context.Context) (CompareDest []fs.Fs, err error) {
ci := fs.GetConfig(ctx)
CompareDest, err = cache.Get(ctx, ci.CompareDest)
CompareDest, err = cache.GetArr(ctx, ci.CompareDest)
if err != nil {
return nil, fserrors.FatalError(errors.Errorf("Failed to make fs for --compare-dest %q: %v", ci.CompareDest, err))
}
@ -1391,18 +1401,21 @@ func compareDest(ctx context.Context, dst, src fs.Object, CompareDest fs.Fs) (No
}
// GetCopyDest sets up --copy-dest
func GetCopyDest(ctx context.Context, fdst fs.Fs) (CopyDest fs.Fs, err error) {
func GetCopyDest(ctx context.Context, fdst fs.Fs) (CopyDest []fs.Fs, err error) {
ci := fs.GetConfig(ctx)
CopyDest, err = cache.Get(ctx, ci.CopyDest)
CopyDest, err = cache.GetArr(ctx, ci.CopyDest)
if err != nil {
return nil, fserrors.FatalError(errors.Errorf("Failed to make fs for --copy-dest %q: %v", ci.CopyDest, err))
}
if !SameConfig(fdst, CopyDest) {
if !SameConfigArr(fdst, CopyDest) {
return nil, fserrors.FatalError(errors.New("parameter to --copy-dest has to be on the same remote as destination"))
}
if CopyDest.Features().Copy == nil {
return nil, fserrors.FatalError(errors.New("can't use --copy-dest on a remote which doesn't support server-side copy"))
for _, cf := range CopyDest {
if cf.Features().Copy == nil {
return nil, fserrors.FatalError(errors.New("can't use --copy-dest on a remote which doesn't support server side copy"))
}
}
return CopyDest, nil
}
@ -1457,12 +1470,22 @@ func copyDest(ctx context.Context, fdst fs.Fs, dst, src fs.Object, CopyDest, bac
// does not need to be copied
//
// Returns True if src does not need to be copied
func CompareOrCopyDest(ctx context.Context, fdst fs.Fs, dst, src fs.Object, CompareOrCopyDest, backupDir fs.Fs) (NoNeedTransfer bool, err error) {
func CompareOrCopyDest(ctx context.Context, fdst fs.Fs, dst, src fs.Object, CompareOrCopyDest []fs.Fs, backupDir fs.Fs) (NoNeedTransfer bool, err error) {
ci := fs.GetConfig(ctx)
if ci.CompareDest != "" {
return compareDest(ctx, dst, src, CompareOrCopyDest)
} else if ci.CopyDest != "" {
return copyDest(ctx, fdst, dst, src, CompareOrCopyDest, backupDir)
if len(ci.CompareDest) > 0 {
for _, compareF := range CompareOrCopyDest {
NoNeedTransfer, err := compareDest(ctx, dst, src, compareF)
if NoNeedTransfer || err != nil {
return NoNeedTransfer, err
}
}
} else if len(ci.CopyDest) > 0 {
for _, copyF := range CompareOrCopyDest {
NoNeedTransfer, err := copyDest(ctx, fdst, dst, src, copyF, backupDir)
if NoNeedTransfer || err != nil {
return NoNeedTransfer, err
}
}
}
return false, nil
}
@ -1732,19 +1755,20 @@ func moveOrCopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName str
return err
}
var backupDir, copyDestDir fs.Fs
var backupDir fs.Fs
var copyDestDir []fs.Fs
if ci.BackupDir != "" || ci.Suffix != "" {
backupDir, err = BackupDir(ctx, fdst, fsrc, srcFileName)
if err != nil {
return errors.Wrap(err, "creating Fs for --backup-dir failed")
}
}
if ci.CompareDest != "" {
if len(ci.CompareDest) > 0 {
copyDestDir, err = GetCompareDest(ctx)
if err != nil {
return err
}
} else if ci.CopyDest != "" {
} else if len(ci.CopyDest) > 0 {
copyDestDir, err = GetCopyDest(ctx, fdst)
if err != nil {
return err

8
fs/operations/operations_test.go
View File

@ -909,9 +909,9 @@ func TestCopyFileCompareDest(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
ci.CompareDest = r.FremoteName + "/CompareDest"
ci.CompareDest = []string{r.FremoteName + "/CompareDest"}
defer func() {
ci.CompareDest = ""
ci.CompareDest = nil
}()
fdst, err := fs.NewFs(ctx, r.FremoteName+"/dst")
require.NoError(t, err)
@ -995,9 +995,9 @@ func TestCopyFileCopyDest(t *testing.T) {
t.Skip("Skipping test as remote does not support server-side copy")
}
ci.CopyDest = r.FremoteName + "/CopyDest"
ci.CopyDest = []string{r.FremoteName + "/CopyDest"}
defer func() {
ci.CopyDest = ""
ci.CopyDest = nil
}()
fdst, err := fs.NewFs(ctx, r.FremoteName+"/dst")

8
fs/sync/sync.go
View File

@ -70,7 +70,7 @@ type syncCopyMove struct {
trackRenamesWg sync.WaitGroup // wg for background track renames
trackRenamesCh chan fs.Object // objects are pumped in here
renameCheck []fs.Object // accumulate files to check for rename here
compareCopyDest fs.Fs // place to check for files to server-side copy
compareCopyDest []fs.Fs // place to check for files to server side copy
backupDir fs.Fs // place to store overwrites/deletes
checkFirst bool // if set run all the checkers before starting transfers
}
@ -212,13 +212,13 @@ func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete
return nil, err
}
}
if ci.CompareDest != "" {
if len(ci.CompareDest) > 0 {
var err error
s.compareCopyDest, err = operations.GetCompareDest(ctx)
if err != nil {
return nil, err
}
} else if ci.CopyDest != "" {
} else if len(ci.CopyDest) > 0 {
var err error
s.compareCopyDest, err = operations.GetCopyDest(ctx, fdst)
if err != nil {
@ -890,7 +890,7 @@ func (s *syncCopyMove) run() error {
// Delete empty fsrc subdirectories
// if DoMove and --delete-empty-src-dirs flag is set
if s.DoMove && s.deleteEmptySrcDirs {
//delete empty subdirectories that were part of the move
// delete empty subdirectories that were part of the move
s.processError(s.deleteEmptyDirectories(s.ctx, s.fsrc, s.srcEmptyDirs))
}

42
fs/sync/sync_test.go
View File

@ -1480,9 +1480,9 @@ func TestSyncCompareDest(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
ci.CompareDest = r.FremoteName + "/CompareDest"
ci.CompareDest = []string{r.FremoteName + "/CompareDest"}
defer func() {
ci.CompareDest = ""
ci.CompareDest = []string{}
}()
fdst, err := fs.NewFs(ctx, r.FremoteName+"/dst")
@ -1562,6 +1562,40 @@ func TestSyncCompareDest(t *testing.T) {
fstest.CheckItems(t, r.Fremote, file2, file3, file4, file5bdst)
}
// Test with multiple CompareDest
func TestSyncMultipleCompareDest(t *testing.T) {
ctx := context.Background()
ci := fs.GetConfig(ctx)
r := fstest.NewRun(t)
defer r.Finalise()
ci.CompareDest = []string{r.FremoteName + "/pre-dest1", r.FremoteName + "/pre-dest2"}
defer func() {
ci.CompareDest = []string{}
}()
// check empty dest, new compare
fsrc1 := r.WriteFile("1", "1", t1)
fsrc2 := r.WriteFile("2", "2", t1)
fsrc3 := r.WriteFile("3", "3", t1)
fstest.CheckItems(t, r.Flocal, fsrc1, fsrc2, fsrc3)
fdest1 := r.WriteObject(ctx, "pre-dest1/1", "1", t1)
fdest2 := r.WriteObject(ctx, "pre-dest2/2", "2", t1)
fstest.CheckItems(t, r.Fremote, fdest1, fdest2)
accounting.GlobalStats().ResetCounters()
fdst, err := fs.NewFs(ctx, r.FremoteName+"/dest")
require.NoError(t, err)
require.NoError(t, Sync(ctx, fdst, r.Flocal, false))
fdest3 := fsrc3
fdest3.Path = "dest/3"
fstest.CheckItems(t, fdst, fsrc3)
fstest.CheckItems(t, r.Fremote, fdest1, fdest2, fdest3)
}
// Test with CopyDest set
func TestSyncCopyDest(t *testing.T) {
ctx := context.Background()
@ -1573,9 +1607,9 @@ func TestSyncCopyDest(t *testing.T) {
t.Skip("Skipping test as remote does not support server-side copy")
}
ci.CopyDest = r.FremoteName + "/CopyDest"
ci.CopyDest = []string{r.FremoteName + "/CopyDest"}
defer func() {
ci.CopyDest = ""
ci.CopyDest = []string{}
}()
fdst, err := fs.NewFs(ctx, r.FremoteName+"/dst")