diff --git a/fs/operations_test.go b/fs/operations_test.go index de7335670..72124c678 100644 --- a/fs/operations_test.go +++ b/fs/operations_test.go @@ -22,16 +22,10 @@ package fs_test import ( "bytes" "errors" - "flag" "fmt" "io" "io/ioutil" - "log" - "os" - "path" - "path/filepath" "regexp" - "sort" "strings" "testing" "time" @@ -52,270 +46,39 @@ var ( // TestMain drives the tests func TestMain(m *testing.M) { - flag.Parse() - if !*fstest.Individual { - oneRun = newRun() - } - rc := m.Run() - if !*fstest.Individual { - oneRun.Finalise() - } - os.Exit(rc) + fstest.TestMain(m) } -// Run holds the remotes for a test run -type Run struct { - localName string - flocal fs.Fs - fremote fs.Fs - fremoteName string - cleanRemote func() - mkdir map[string]bool // whether the remote has been made yet for the fs name - Logf, Fatalf func(text string, args ...interface{}) -} - -// oneRun holds the master run data if individual is not set -var oneRun *Run - -// newRun initialise the remote and local for testing and returns a -// run object. -// -// r.flocal is an empty local Fs -// r.fremote is an empty remote Fs -// -// Finalise() will tidy them away when done. -func newRun() *Run { - r := &Run{ - Logf: log.Printf, - Fatalf: log.Fatalf, - mkdir: make(map[string]bool), - } - - fstest.Initialise() - - var err error - r.fremote, r.fremoteName, r.cleanRemote, err = fstest.RandomRemote(*fstest.RemoteName, *fstest.SubDir) - if err != nil { - r.Fatalf("Failed to open remote %q: %v", *fstest.RemoteName, err) - } - - r.localName, err = ioutil.TempDir("", "rclone") - if err != nil { - r.Fatalf("Failed to create temp dir: %v", err) - } - r.localName = filepath.ToSlash(r.localName) - r.flocal, err = fs.NewFs(r.localName) - if err != nil { - r.Fatalf("Failed to make %q: %v", r.localName, err) - } - fs.CalculateModifyWindow(r.fremote, r.flocal) - return r -} - -// dirsToRemove sorts by string length -type dirsToRemove []string - -func (d dirsToRemove) Len() int { return len(d) } -func (d dirsToRemove) Swap(i, j int) { d[i], d[j] = d[j], d[i] } -func (d dirsToRemove) Less(i, j int) bool { return len(d[i]) > len(d[j]) } - -// NewRun initialise the remote and local for testing and returns a -// run object. Call this from the tests. -// -// r.flocal is an empty local Fs -// r.fremote is an empty remote Fs -// -// Finalise() will tidy them away when done. -func NewRun(t *testing.T) *Run { - var r *Run - if *fstest.Individual { - r = newRun() - } else { - // If not individual, use the global one with the clean method overridden - r = new(Run) - *r = *oneRun - r.cleanRemote = func() { - var toDelete dirsToRemove - require.NoError(t, fs.Walk(r.fremote, "", true, -1, func(dirPath string, entries fs.DirEntries, err error) error { - if err != nil { - if err == fs.ErrorDirNotFound { - return nil - } - t.Fatalf("Error listing: %v", err) - } - for _, entry := range entries { - switch x := entry.(type) { - case fs.Object: - err = x.Remove() - if err != nil { - t.Errorf("Error removing file %q: %v", x.Remote(), err) - } - case fs.Directory: - toDelete = append(toDelete, x.Remote()) - } - } - return nil - })) - sort.Sort(toDelete) - for _, dir := range toDelete { - err := r.fremote.Rmdir(dir) - if err != nil { - t.Errorf("Error removing dir %q: %v", dir, err) - } - } - // Check remote is empty - fstest.CheckItems(t, r.fremote) - } - } - r.Logf = t.Logf - r.Fatalf = t.Fatalf - r.Logf("Remote %q, Local %q, Modify Window %q", r.fremote, r.flocal, fs.Config.ModifyWindow) - return r -} - -// Rename a file in local -func (r *Run) RenameFile(item fstest.Item, newpath string) fstest.Item { - oldFilepath := path.Join(r.localName, item.Path) - newFilepath := path.Join(r.localName, newpath) - if err := os.Rename(oldFilepath, newFilepath); err != nil { - r.Fatalf("Failed to rename file from %q to %q: %v", item.Path, newpath, err) - } - - item.Path = newpath - - return item -} - -// Write a file to local -func (r *Run) WriteFile(filePath, content string, t time.Time) fstest.Item { - item := fstest.NewItem(filePath, content, t) - // FIXME make directories? - filePath = path.Join(r.localName, filePath) - dirPath := path.Dir(filePath) - err := os.MkdirAll(dirPath, 0770) - if err != nil { - r.Fatalf("Failed to make directories %q: %v", dirPath, err) - } - err = ioutil.WriteFile(filePath, []byte(content), 0600) - if err != nil { - r.Fatalf("Failed to write file %q: %v", filePath, err) - } - err = os.Chtimes(filePath, t, t) - if err != nil { - r.Fatalf("Failed to chtimes file %q: %v", filePath, err) - } - return item -} - -// ForceMkdir creates the remote -func (r *Run) ForceMkdir(f fs.Fs) { - err := f.Mkdir("") - if err != nil { - r.Fatalf("Failed to mkdir %q: %v", f, err) - } - r.mkdir[f.String()] = true -} - -// Mkdir creates the remote if it hasn't been created already -func (r *Run) Mkdir(f fs.Fs) { - if !r.mkdir[f.String()] { - r.ForceMkdir(f) - } -} - -// WriteObjectTo writes an object to the fs, remote passed in -func (r *Run) WriteObjectTo(f fs.Fs, remote, content string, modTime time.Time, useUnchecked bool) fstest.Item { - put := f.Put - if useUnchecked { - put = f.Features().PutUnchecked - if put == nil { - r.Fatalf("Fs doesn't support PutUnchecked") - } - } - r.Mkdir(f) - const maxTries = 10 - for tries := 1; ; tries++ { - in := bytes.NewBufferString(content) - objinfo := fs.NewStaticObjectInfo(remote, modTime, int64(len(content)), true, nil, nil) - _, err := put(in, objinfo) - if err == nil { - break - } - // Retry if err returned a retry error - if fs.IsRetryError(err) && tries < maxTries { - r.Logf("Retry Put of %q to %v: %d/%d (%v)", remote, f, tries, maxTries, err) - time.Sleep(2 * time.Second) - continue - } - r.Fatalf("Failed to put %q to %q: %v", remote, f, err) - } - return fstest.NewItem(remote, content, modTime) -} - -// WriteObject writes an object to the remote -func (r *Run) WriteObject(remote, content string, modTime time.Time) fstest.Item { - return r.WriteObjectTo(r.fremote, remote, content, modTime, false) -} - -// WriteUncheckedObject writes an object to the remote not checking for duplicates -func (r *Run) WriteUncheckedObject(remote, content string, modTime time.Time) fstest.Item { - return r.WriteObjectTo(r.fremote, remote, content, modTime, true) -} - -// WriteBoth calls WriteObject and WriteFile with the same arguments -func (r *Run) WriteBoth(remote, content string, modTime time.Time) fstest.Item { - r.WriteFile(remote, content, modTime) - return r.WriteObject(remote, content, modTime) -} - -// Clean the temporary directory -func (r *Run) cleanTempDir() { - err := os.RemoveAll(r.localName) - if err != nil { - r.Logf("Failed to clean temporary directory %q: %v", r.localName, err) - } -} - -// finalise cleans the remote and local -func (r *Run) Finalise() { - // r.Logf("Cleaning remote %q", r.fremote) - r.cleanRemote() - // r.Logf("Cleaning local %q", r.localName) - r.cleanTempDir() -} - -// ------------------------------------------------------------ - func TestMkdir(t *testing.T) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() - fstest.TestMkdir(t, r.fremote) + fstest.TestMkdir(t, r.Fremote) } func TestLsd(t *testing.T) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() file1 := r.WriteObject("sub dir/hello world", "hello world", t1) - fstest.CheckItems(t, r.fremote, file1) + fstest.CheckItems(t, r.Fremote, file1) var buf bytes.Buffer - err := fs.ListDir(r.fremote, &buf) + err := fs.ListDir(r.Fremote, &buf) require.NoError(t, err) res := buf.String() assert.Contains(t, res, "sub dir\n") } func TestLs(t *testing.T) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() file1 := r.WriteBoth("potato2", "------------------------------------------------------------", t1) file2 := r.WriteBoth("empty space", "", t2) - fstest.CheckItems(t, r.fremote, file1, file2) + fstest.CheckItems(t, r.Fremote, file1, file2) var buf bytes.Buffer - err := fs.List(r.fremote, &buf) + err := fs.List(r.Fremote, &buf) require.NoError(t, err) res := buf.String() assert.Contains(t, res, " 0 empty space\n") @@ -323,22 +86,22 @@ func TestLs(t *testing.T) { } func TestLsLong(t *testing.T) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() file1 := r.WriteBoth("potato2", "------------------------------------------------------------", t1) file2 := r.WriteBoth("empty space", "", t2) - fstest.CheckItems(t, r.fremote, file1, file2) + fstest.CheckItems(t, r.Fremote, file1, file2) var buf bytes.Buffer - err := fs.ListLong(r.fremote, &buf) + err := fs.ListLong(r.Fremote, &buf) require.NoError(t, err) res := buf.String() lines := strings.Split(strings.Trim(res, "\n"), "\n") assert.Equal(t, 2, len(lines)) timeFormat := "2006-01-02 15:04:05.000000000" - precision := r.fremote.Precision() + precision := r.Fremote.Precision() location := time.Now().Location() checkTime := func(m, filename string, expected time.Time) { modTime, err := time.ParseInLocation(timeFormat, m, location) // parse as localtime @@ -368,17 +131,17 @@ func TestLsLong(t *testing.T) { } func TestHashSums(t *testing.T) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() file1 := r.WriteBoth("potato2", "------------------------------------------------------------", t1) file2 := r.WriteBoth("empty space", "", t2) - fstest.CheckItems(t, r.fremote, file1, file2) + fstest.CheckItems(t, r.Fremote, file1, file2) // MD5 Sum var buf bytes.Buffer - err := fs.Md5sum(r.fremote, &buf) + err := fs.Md5sum(r.Fremote, &buf) require.NoError(t, err) res := buf.String() if !strings.Contains(res, "d41d8cd98f00b204e9800998ecf8427e empty space\n") && @@ -395,7 +158,7 @@ func TestHashSums(t *testing.T) { // SHA1 Sum buf.Reset() - err = fs.Sha1sum(r.fremote, &buf) + err = fs.Sha1sum(r.Fremote, &buf) require.NoError(t, err) res = buf.String() if !strings.Contains(res, "da39a3ee5e6b4b0d3255bfef95601890afd80709 empty space\n") && @@ -412,7 +175,7 @@ func TestHashSums(t *testing.T) { // Dropbox Hash Sum buf.Reset() - err = fs.DropboxHashSum(r.fremote, &buf) + err = fs.DropboxHashSum(r.Fremote, &buf) require.NoError(t, err) res = buf.String() if !strings.Contains(res, "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 empty space\n") && @@ -428,50 +191,50 @@ func TestHashSums(t *testing.T) { } func TestCount(t *testing.T) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() file1 := r.WriteBoth("potato2", "------------------------------------------------------------", t1) file2 := r.WriteBoth("empty space", "", t2) file3 := r.WriteBoth("sub dir/potato3", "hello", t2) - fstest.CheckItems(t, r.fremote, file1, file2, file3) + fstest.CheckItems(t, r.Fremote, file1, file2, file3) // Check the MaxDepth too fs.Config.MaxDepth = 1 defer func() { fs.Config.MaxDepth = -1 }() - objects, size, err := fs.Count(r.fremote) + objects, size, err := fs.Count(r.Fremote) require.NoError(t, err) assert.Equal(t, int64(2), objects) assert.Equal(t, int64(60), size) } func TestDelete(t *testing.T) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() file1 := r.WriteObject("small", "1234567890", t2) // 10 bytes file2 := r.WriteObject("medium", "------------------------------------------------------------", t1) // 60 bytes file3 := r.WriteObject("large", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 100 bytes - fstest.CheckItems(t, r.fremote, file1, file2, file3) + fstest.CheckItems(t, r.Fremote, file1, file2, file3) fs.Config.Filter.MaxSize = 60 defer func() { fs.Config.Filter.MaxSize = -1 }() - err := fs.Delete(r.fremote) + err := fs.Delete(r.Fremote) require.NoError(t, err) - fstest.CheckItems(t, r.fremote, file3) + fstest.CheckItems(t, r.Fremote, file3) } func testCheck(t *testing.T, checkFunction func(fdst, fsrc fs.Fs) error) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() check := func(i int, wantErrors int64) { - fs.Debugf(r.fremote, "%d: Starting check test", i) + fs.Debugf(r.Fremote, "%d: Starting check test", i) oldErrors := fs.Stats.GetErrors() - err := checkFunction(r.flocal, r.fremote) + err := checkFunction(r.Flocal, r.Fremote) gotErrors := fs.Stats.GetErrors() - oldErrors if wantErrors == 0 && err != nil { t.Errorf("%d: Got error when not expecting one: %v", i, err) @@ -482,20 +245,20 @@ func testCheck(t *testing.T, checkFunction func(fdst, fsrc fs.Fs) error) { if wantErrors != gotErrors { t.Errorf("%d: Expecting %d errors but got %d", i, wantErrors, gotErrors) } - fs.Debugf(r.fremote, "%d: Ending check test", i) + fs.Debugf(r.Fremote, "%d: Ending check test", i) } file1 := r.WriteBoth("rutabaga", "is tasty", t3) - fstest.CheckItems(t, r.fremote, file1) - fstest.CheckItems(t, r.flocal, file1) + fstest.CheckItems(t, r.Fremote, file1) + fstest.CheckItems(t, r.Flocal, file1) check(1, 0) file2 := r.WriteFile("potato2", "------------------------------------------------------------", t1) - fstest.CheckItems(t, r.flocal, file1, file2) + fstest.CheckItems(t, r.Flocal, file1, file2) check(2, 1) file3 := r.WriteObject("empty space", "", t2) - fstest.CheckItems(t, r.fremote, file1, file3) + fstest.CheckItems(t, r.Fremote, file1, file3) check(3, 2) file2r := file2 @@ -504,11 +267,11 @@ func testCheck(t *testing.T, checkFunction func(fdst, fsrc fs.Fs) error) { } else { r.WriteObject("potato2", "------------------------------------------------------------", t1) } - fstest.CheckItems(t, r.fremote, file1, file2r, file3) + fstest.CheckItems(t, r.Fremote, file1, file2r, file3) check(4, 1) r.WriteFile("empty space", "", t2) - fstest.CheckItems(t, r.flocal, file1, file2, file3) + fstest.CheckItems(t, r.Flocal, file1, file2, file3) check(5, 0) } @@ -526,17 +289,6 @@ func TestCheckSizeOnly(t *testing.T) { TestCheck(t) } -func (r *Run) checkWithDuplicates(t *testing.T, items ...fstest.Item) { - objects, size, err := fs.Count(r.fremote) - require.NoError(t, err) - assert.Equal(t, int64(len(items)), objects) - wantSize := int64(0) - for _, item := range items { - wantSize += item.Size - } - assert.Equal(t, wantSize, size) -} - func skipIfCantDedupe(t *testing.T, f fs.Fs) { if f.Features().PutUnchecked == nil { t.Skip("Can't test deduplicate - no PutUnchecked") @@ -550,51 +302,51 @@ func skipIfCantDedupe(t *testing.T, f fs.Fs) { } func TestDeduplicateInteractive(t *testing.T) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() - skipIfCantDedupe(t, r.fremote) + skipIfCantDedupe(t, r.Fremote) file1 := r.WriteUncheckedObject("one", "This is one", t1) file2 := r.WriteUncheckedObject("one", "This is one", t1) file3 := r.WriteUncheckedObject("one", "This is one", t1) - r.checkWithDuplicates(t, file1, file2, file3) + r.CheckWithDuplicates(t, file1, file2, file3) - err := fs.Deduplicate(r.fremote, fs.DeduplicateInteractive) + err := fs.Deduplicate(r.Fremote, fs.DeduplicateInteractive) require.NoError(t, err) - fstest.CheckItems(t, r.fremote, file1) + fstest.CheckItems(t, r.Fremote, file1) } func TestDeduplicateSkip(t *testing.T) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() - skipIfCantDedupe(t, r.fremote) + skipIfCantDedupe(t, r.Fremote) file1 := r.WriteUncheckedObject("one", "This is one", t1) file2 := r.WriteUncheckedObject("one", "This is one", t1) file3 := r.WriteUncheckedObject("one", "This is another one", t1) - r.checkWithDuplicates(t, file1, file2, file3) + r.CheckWithDuplicates(t, file1, file2, file3) - err := fs.Deduplicate(r.fremote, fs.DeduplicateSkip) + err := fs.Deduplicate(r.Fremote, fs.DeduplicateSkip) require.NoError(t, err) - r.checkWithDuplicates(t, file1, file3) + r.CheckWithDuplicates(t, file1, file3) } func TestDeduplicateFirst(t *testing.T) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() - skipIfCantDedupe(t, r.fremote) + skipIfCantDedupe(t, r.Fremote) file1 := r.WriteUncheckedObject("one", "This is one", t1) file2 := r.WriteUncheckedObject("one", "This is one A", t1) file3 := r.WriteUncheckedObject("one", "This is one BB", t1) - r.checkWithDuplicates(t, file1, file2, file3) + r.CheckWithDuplicates(t, file1, file2, file3) - err := fs.Deduplicate(r.fremote, fs.DeduplicateFirst) + err := fs.Deduplicate(r.Fremote, fs.DeduplicateFirst) require.NoError(t, err) - objects, size, err := fs.Count(r.fremote) + objects, size, err := fs.Count(r.Fremote) require.NoError(t, err) assert.Equal(t, int64(1), objects) if size != file1.Size && size != file2.Size && size != file3.Size { @@ -603,51 +355,51 @@ func TestDeduplicateFirst(t *testing.T) { } func TestDeduplicateNewest(t *testing.T) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() - skipIfCantDedupe(t, r.fremote) + skipIfCantDedupe(t, r.Fremote) file1 := r.WriteUncheckedObject("one", "This is one", t1) file2 := r.WriteUncheckedObject("one", "This is one too", t2) file3 := r.WriteUncheckedObject("one", "This is another one", t3) - r.checkWithDuplicates(t, file1, file2, file3) + r.CheckWithDuplicates(t, file1, file2, file3) - err := fs.Deduplicate(r.fremote, fs.DeduplicateNewest) + err := fs.Deduplicate(r.Fremote, fs.DeduplicateNewest) require.NoError(t, err) - fstest.CheckItems(t, r.fremote, file3) + fstest.CheckItems(t, r.Fremote, file3) } func TestDeduplicateOldest(t *testing.T) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() - skipIfCantDedupe(t, r.fremote) + skipIfCantDedupe(t, r.Fremote) file1 := r.WriteUncheckedObject("one", "This is one", t1) file2 := r.WriteUncheckedObject("one", "This is one too", t2) file3 := r.WriteUncheckedObject("one", "This is another one", t3) - r.checkWithDuplicates(t, file1, file2, file3) + r.CheckWithDuplicates(t, file1, file2, file3) - err := fs.Deduplicate(r.fremote, fs.DeduplicateOldest) + err := fs.Deduplicate(r.Fremote, fs.DeduplicateOldest) require.NoError(t, err) - fstest.CheckItems(t, r.fremote, file1) + fstest.CheckItems(t, r.Fremote, file1) } func TestDeduplicateRename(t *testing.T) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() - skipIfCantDedupe(t, r.fremote) + skipIfCantDedupe(t, r.Fremote) file1 := r.WriteUncheckedObject("one.txt", "This is one", t1) file2 := r.WriteUncheckedObject("one.txt", "This is one too", t2) file3 := r.WriteUncheckedObject("one.txt", "This is another one", t3) - r.checkWithDuplicates(t, file1, file2, file3) + r.CheckWithDuplicates(t, file1, file2, file3) - err := fs.Deduplicate(r.fremote, fs.DeduplicateRename) + err := fs.Deduplicate(r.Fremote, fs.DeduplicateRename) require.NoError(t, err) - require.NoError(t, fs.Walk(r.fremote, "", true, -1, func(dirPath string, entries fs.DirEntries, err error) error { + require.NoError(t, fs.Walk(r.Fremote, "", true, -1, func(dirPath string, entries fs.DirEntries, err error) error { if err != nil { return err } @@ -670,10 +422,10 @@ func TestDeduplicateRename(t *testing.T) { // This should really be a unit test, but the test framework there // doesn't have enough tools to make it easy func TestMergeDirs(t *testing.T) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() - mergeDirs := r.fremote.Features().MergeDirs + mergeDirs := r.Fremote.Features().MergeDirs if mergeDirs == nil { t.Skip("Can't merge directories") } @@ -682,7 +434,7 @@ func TestMergeDirs(t *testing.T) { file2 := r.WriteObject("dupe2/two.txt", "This is one too", t2) file3 := r.WriteObject("dupe3/three.txt", "This is another one", t3) - objs, dirs, err := fs.WalkGetAll(r.fremote, "", true, 1) + objs, dirs, err := fs.WalkGetAll(r.Fremote, "", true, 1) require.NoError(t, err) assert.Equal(t, 3, len(dirs)) assert.Equal(t, 0, len(objs)) @@ -692,9 +444,9 @@ func TestMergeDirs(t *testing.T) { file2.Path = "dupe1/two.txt" file3.Path = "dupe1/three.txt" - fstest.CheckItems(t, r.fremote, file1, file2, file3) + fstest.CheckItems(t, r.Fremote, file1, file2, file3) - objs, dirs, err = fs.WalkGetAll(r.fremote, "", true, 1) + objs, dirs, err = fs.WalkGetAll(r.Fremote, "", true, 1) require.NoError(t, err) assert.Equal(t, 1, len(dirs)) assert.Equal(t, 0, len(objs)) @@ -702,12 +454,12 @@ func TestMergeDirs(t *testing.T) { } func TestCat(t *testing.T) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() file1 := r.WriteBoth("file1", "ABCDEFGHIJ", t1) file2 := r.WriteBoth("file2", "012345678", t2) - fstest.CheckItems(t, r.fremote, file1, file2) + fstest.CheckItems(t, r.Fremote, file1, file2) for _, test := range []struct { offset int64 @@ -721,7 +473,7 @@ func TestCat(t *testing.T) { {1, 3, "BCD", "123"}, } { var buf bytes.Buffer - err := fs.Cat(r.fremote, &buf, test.offset, test.count) + err := fs.Cat(r.Fremote, &buf, test.offset, test.count) require.NoError(t, err) res := buf.String() @@ -742,10 +494,10 @@ func TestRcat(t *testing.T) { prefix = "with_checksum_" } - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() - fstest.CheckListing(t, r.fremote, []fstest.Item{}) + fstest.CheckListing(t, r.Fremote, []fstest.Item{}) data1 := "this is some really nice test data" path1 := prefix + "small_file_from_pipe" @@ -754,16 +506,16 @@ func TestRcat(t *testing.T) { path2 := prefix + "big_file_from_pipe" in := ioutil.NopCloser(strings.NewReader(data1)) - _, err := fs.Rcat(r.fremote, path1, in, t1) + _, err := fs.Rcat(r.Fremote, path1, in, t1) require.NoError(t, err) in = ioutil.NopCloser(strings.NewReader(data2)) - _, err = fs.Rcat(r.fremote, path2, in, t2) + _, err = fs.Rcat(r.Fremote, path2, in, t2) require.NoError(t, err) file1 := fstest.NewItem(path1, data1, t1) file2 := fstest.NewItem(path2, data2, t2) - fstest.CheckItems(t, r.fremote, file1, file2) + fstest.CheckItems(t, r.Fremote, file1, file2) } check(true) @@ -771,27 +523,27 @@ func TestRcat(t *testing.T) { } func TestRmdirs(t *testing.T) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() - r.Mkdir(r.fremote) + r.Mkdir(r.Fremote) // Make some files and dirs we expect to keep - r.ForceMkdir(r.fremote) + r.ForceMkdir(r.Fremote) file1 := r.WriteObject("A1/B1/C1/one", "aaa", t1) //..and dirs we expect to delete - require.NoError(t, fs.Mkdir(r.fremote, "A2")) - require.NoError(t, fs.Mkdir(r.fremote, "A1/B2")) - require.NoError(t, fs.Mkdir(r.fremote, "A1/B2/C2")) - require.NoError(t, fs.Mkdir(r.fremote, "A1/B1/C3")) - require.NoError(t, fs.Mkdir(r.fremote, "A3")) - require.NoError(t, fs.Mkdir(r.fremote, "A3/B3")) - require.NoError(t, fs.Mkdir(r.fremote, "A3/B3/C4")) + require.NoError(t, fs.Mkdir(r.Fremote, "A2")) + require.NoError(t, fs.Mkdir(r.Fremote, "A1/B2")) + require.NoError(t, fs.Mkdir(r.Fremote, "A1/B2/C2")) + require.NoError(t, fs.Mkdir(r.Fremote, "A1/B1/C3")) + require.NoError(t, fs.Mkdir(r.Fremote, "A3")) + require.NoError(t, fs.Mkdir(r.Fremote, "A3/B3")) + require.NoError(t, fs.Mkdir(r.Fremote, "A3/B3/C4")) //..and one more file at the end file2 := r.WriteObject("A1/two", "bbb", t2) fstest.CheckListingWithPrecision( t, - r.fremote, + r.Fremote, []fstest.Item{ file1, file2, }, @@ -810,11 +562,11 @@ func TestRmdirs(t *testing.T) { fs.Config.ModifyWindow, ) - require.NoError(t, fs.Rmdirs(r.fremote, "")) + require.NoError(t, fs.Rmdirs(r.Fremote, "")) fstest.CheckListingWithPrecision( t, - r.fremote, + r.Fremote, []fstest.Item{ file1, file2, }, @@ -829,58 +581,58 @@ func TestRmdirs(t *testing.T) { } func TestMoveFile(t *testing.T) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() file1 := r.WriteFile("file1", "file1 contents", t1) - fstest.CheckItems(t, r.flocal, file1) + fstest.CheckItems(t, r.Flocal, file1) file2 := file1 file2.Path = "sub/file2" - err := fs.MoveFile(r.fremote, r.flocal, file2.Path, file1.Path) + err := fs.MoveFile(r.Fremote, r.Flocal, file2.Path, file1.Path) require.NoError(t, err) - fstest.CheckItems(t, r.flocal) - fstest.CheckItems(t, r.fremote, file2) + fstest.CheckItems(t, r.Flocal) + fstest.CheckItems(t, r.Fremote, file2) r.WriteFile("file1", "file1 contents", t1) - fstest.CheckItems(t, r.flocal, file1) + fstest.CheckItems(t, r.Flocal, file1) - err = fs.MoveFile(r.fremote, r.flocal, file2.Path, file1.Path) + err = fs.MoveFile(r.Fremote, r.Flocal, file2.Path, file1.Path) require.NoError(t, err) - fstest.CheckItems(t, r.flocal) - fstest.CheckItems(t, r.fremote, file2) + fstest.CheckItems(t, r.Flocal) + fstest.CheckItems(t, r.Fremote, file2) - err = fs.MoveFile(r.fremote, r.fremote, file2.Path, file2.Path) + err = fs.MoveFile(r.Fremote, r.Fremote, file2.Path, file2.Path) require.NoError(t, err) - fstest.CheckItems(t, r.flocal) - fstest.CheckItems(t, r.fremote, file2) + fstest.CheckItems(t, r.Flocal) + fstest.CheckItems(t, r.Fremote, file2) } func TestCopyFile(t *testing.T) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() file1 := r.WriteFile("file1", "file1 contents", t1) - fstest.CheckItems(t, r.flocal, file1) + fstest.CheckItems(t, r.Flocal, file1) file2 := file1 file2.Path = "sub/file2" - err := fs.CopyFile(r.fremote, r.flocal, file2.Path, file1.Path) + err := fs.CopyFile(r.Fremote, r.Flocal, file2.Path, file1.Path) require.NoError(t, err) - fstest.CheckItems(t, r.flocal, file1) - fstest.CheckItems(t, r.fremote, file2) + fstest.CheckItems(t, r.Flocal, file1) + fstest.CheckItems(t, r.Fremote, file2) - err = fs.CopyFile(r.fremote, r.flocal, file2.Path, file1.Path) + err = fs.CopyFile(r.Fremote, r.Flocal, file2.Path, file1.Path) require.NoError(t, err) - fstest.CheckItems(t, r.flocal, file1) - fstest.CheckItems(t, r.fremote, file2) + fstest.CheckItems(t, r.Flocal, file1) + fstest.CheckItems(t, r.Fremote, file2) - err = fs.CopyFile(r.fremote, r.fremote, file2.Path, file2.Path) + err = fs.CopyFile(r.Fremote, r.Fremote, file2.Path, file2.Path) require.NoError(t, err) - fstest.CheckItems(t, r.flocal, file1) - fstest.CheckItems(t, r.fremote, file2) + fstest.CheckItems(t, r.Flocal, file1) + fstest.CheckItems(t, r.Fremote, file2) } // testFsInfo is for unit testing fs.Info @@ -978,7 +730,7 @@ func TestOverlapping(t *testing.T) { } func TestListDirSorted(t *testing.T) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() fs.Config.Filter.MaxSize = 10 @@ -993,7 +745,7 @@ func TestListDirSorted(t *testing.T) { r.WriteObject("sub dir/hello world2", "hello world", t1), r.WriteObject("sub dir/sub sub dir/hello world3", "hello world", t1), } - fstest.CheckItems(t, r.fremote, files...) + fstest.CheckItems(t, r.Fremote, files...) var items fs.DirEntries var err error @@ -1012,27 +764,27 @@ func TestListDirSorted(t *testing.T) { return name } - items, err = fs.ListDirSorted(r.fremote, true, "") + items, err = fs.ListDirSorted(r.Fremote, true, "") require.NoError(t, err) require.Len(t, items, 3) assert.Equal(t, "a.txt", str(0)) assert.Equal(t, "sub dir/", str(1)) assert.Equal(t, "zend.txt", str(2)) - items, err = fs.ListDirSorted(r.fremote, false, "") + items, err = fs.ListDirSorted(r.Fremote, false, "") require.NoError(t, err) require.Len(t, items, 2) assert.Equal(t, "sub dir/", str(0)) assert.Equal(t, "zend.txt", str(1)) - items, err = fs.ListDirSorted(r.fremote, true, "sub dir") + items, err = fs.ListDirSorted(r.Fremote, true, "sub dir") require.NoError(t, err) require.Len(t, items, 3) assert.Equal(t, "sub dir/hello world", str(0)) assert.Equal(t, "sub dir/hello world2", str(1)) assert.Equal(t, "sub dir/sub sub dir/", str(2)) - items, err = fs.ListDirSorted(r.fremote, false, "sub dir") + items, err = fs.ListDirSorted(r.Fremote, false, "sub dir") require.NoError(t, err) require.Len(t, items, 1) assert.Equal(t, "sub dir/sub sub dir/", str(0)) diff --git a/fs/sync_test.go b/fs/sync_test.go index 1335179c9..da34a6d22 100644 --- a/fs/sync_test.go +++ b/fs/sync_test.go @@ -16,37 +16,37 @@ import ( // Check dry run is working func TestCopyWithDryRun(t *testing.T) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() file1 := r.WriteFile("sub dir/hello world", "hello world", t1) - r.Mkdir(r.fremote) + r.Mkdir(r.Fremote) fs.Config.DryRun = true - err := fs.CopyDir(r.fremote, r.flocal) + err := fs.CopyDir(r.Fremote, r.Flocal) fs.Config.DryRun = false require.NoError(t, err) - fstest.CheckItems(t, r.flocal, file1) - fstest.CheckItems(t, r.fremote) + fstest.CheckItems(t, r.Flocal, file1) + fstest.CheckItems(t, r.Fremote) } // Now without dry run func TestCopy(t *testing.T) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() file1 := r.WriteFile("sub dir/hello world", "hello world", t1) - r.Mkdir(r.fremote) + r.Mkdir(r.Fremote) - err := fs.CopyDir(r.fremote, r.flocal) + err := fs.CopyDir(r.Fremote, r.Flocal) require.NoError(t, err) - fstest.CheckItems(t, r.flocal, file1) - fstest.CheckItems(t, r.fremote, file1) + fstest.CheckItems(t, r.Flocal, file1) + fstest.CheckItems(t, r.Fremote, file1) } // Now with --no-traverse func TestCopyNoTraverse(t *testing.T) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() fs.Config.NoTraverse = true @@ -54,16 +54,16 @@ func TestCopyNoTraverse(t *testing.T) { file1 := r.WriteFile("sub dir/hello world", "hello world", t1) - err := fs.CopyDir(r.fremote, r.flocal) + err := fs.CopyDir(r.Fremote, r.Flocal) require.NoError(t, err) - fstest.CheckItems(t, r.flocal, file1) - fstest.CheckItems(t, r.fremote, file1) + fstest.CheckItems(t, r.Flocal, file1) + fstest.CheckItems(t, r.Fremote, file1) } // Now with --no-traverse func TestSyncNoTraverse(t *testing.T) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() fs.Config.NoTraverse = true @@ -72,16 +72,16 @@ func TestSyncNoTraverse(t *testing.T) { file1 := r.WriteFile("sub dir/hello world", "hello world", t1) fs.Stats.ResetCounters() - err := fs.Sync(r.fremote, r.flocal) + err := fs.Sync(r.Fremote, r.Flocal) require.NoError(t, err) - fstest.CheckItems(t, r.flocal, file1) - fstest.CheckItems(t, r.fremote, file1) + fstest.CheckItems(t, r.Flocal, file1) + fstest.CheckItems(t, r.Fremote, file1) } // Test copy with depth func TestCopyWithDepth(t *testing.T) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() file1 := r.WriteFile("sub dir/hello world", "hello world", t1) file2 := r.WriteFile("hello world2", "hello world2", t2) @@ -90,173 +90,173 @@ func TestCopyWithDepth(t *testing.T) { fs.Config.MaxDepth = 1 defer func() { fs.Config.MaxDepth = -1 }() - err := fs.CopyDir(r.fremote, r.flocal) + err := fs.CopyDir(r.Fremote, r.Flocal) require.NoError(t, err) - fstest.CheckItems(t, r.flocal, file1, file2) - fstest.CheckItems(t, r.fremote, file2) + fstest.CheckItems(t, r.Flocal, file1, file2) + fstest.CheckItems(t, r.Fremote, file2) } // Test a server side copy if possible, or the backup path if not func TestServerSideCopy(t *testing.T) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() file1 := r.WriteObject("sub dir/hello world", "hello world", t1) - fstest.CheckItems(t, r.fremote, file1) + fstest.CheckItems(t, r.Fremote, file1) - fremoteCopy, _, finaliseCopy, err := fstest.RandomRemote(*fstest.RemoteName, *fstest.SubDir) + FremoteCopy, _, finaliseCopy, err := fstest.RandomRemote(*fstest.RemoteName, *fstest.SubDir) require.NoError(t, err) defer finaliseCopy() - t.Logf("Server side copy (if possible) %v -> %v", r.fremote, fremoteCopy) + t.Logf("Server side copy (if possible) %v -> %v", r.Fremote, FremoteCopy) - err = fs.CopyDir(fremoteCopy, r.fremote) + err = fs.CopyDir(FremoteCopy, r.Fremote) require.NoError(t, err) - fstest.CheckItems(t, fremoteCopy, file1) + fstest.CheckItems(t, FremoteCopy, file1) } // Check that if the local file doesn't exist when we copy it up, // nothing happens to the remote file func TestCopyAfterDelete(t *testing.T) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() file1 := r.WriteObject("sub dir/hello world", "hello world", t1) - fstest.CheckItems(t, r.flocal) - fstest.CheckItems(t, r.fremote, file1) + fstest.CheckItems(t, r.Flocal) + fstest.CheckItems(t, r.Fremote, file1) - err := fs.Mkdir(r.flocal, "") + err := fs.Mkdir(r.Flocal, "") require.NoError(t, err) - err = fs.CopyDir(r.fremote, r.flocal) + err = fs.CopyDir(r.Fremote, r.Flocal) require.NoError(t, err) - fstest.CheckItems(t, r.flocal) - fstest.CheckItems(t, r.fremote, file1) + fstest.CheckItems(t, r.Flocal) + fstest.CheckItems(t, r.Fremote, file1) } // Check the copy downloading a file func TestCopyRedownload(t *testing.T) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() file1 := r.WriteObject("sub dir/hello world", "hello world", t1) - fstest.CheckItems(t, r.fremote, file1) + fstest.CheckItems(t, r.Fremote, file1) - err := fs.CopyDir(r.flocal, r.fremote) + err := fs.CopyDir(r.Flocal, r.Fremote) require.NoError(t, err) - fstest.CheckItems(t, r.flocal, file1) + fstest.CheckItems(t, r.Flocal, file1) } // Create a file and sync it. Change the last modified date and resync. // If we're only doing sync by size and checksum, we expect nothing to // to be transferred on the second sync. func TestSyncBasedOnCheckSum(t *testing.T) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() fs.Config.CheckSum = true defer func() { fs.Config.CheckSum = false }() file1 := r.WriteFile("check sum", "", t1) - fstest.CheckItems(t, r.flocal, file1) + fstest.CheckItems(t, r.Flocal, file1) fs.Stats.ResetCounters() - err := fs.Sync(r.fremote, r.flocal) + err := fs.Sync(r.Fremote, r.Flocal) require.NoError(t, err) // We should have transferred exactly one file. assert.Equal(t, int64(1), fs.Stats.GetTransfers()) - fstest.CheckItems(t, r.fremote, file1) + fstest.CheckItems(t, r.Fremote, file1) // Change last modified date only file2 := r.WriteFile("check sum", "", t2) - fstest.CheckItems(t, r.flocal, file2) + fstest.CheckItems(t, r.Flocal, file2) fs.Stats.ResetCounters() - err = fs.Sync(r.fremote, r.flocal) + err = fs.Sync(r.Fremote, r.Flocal) require.NoError(t, err) // We should have transferred no files assert.Equal(t, int64(0), fs.Stats.GetTransfers()) - fstest.CheckItems(t, r.flocal, file2) - fstest.CheckItems(t, r.fremote, file1) + fstest.CheckItems(t, r.Flocal, file2) + fstest.CheckItems(t, r.Fremote, file1) } // Create a file and sync it. Change the last modified date and the // file contents but not the size. If we're only doing sync by size // only, we expect nothing to to be transferred on the second sync. func TestSyncSizeOnly(t *testing.T) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() fs.Config.SizeOnly = true defer func() { fs.Config.SizeOnly = false }() file1 := r.WriteFile("sizeonly", "potato", t1) - fstest.CheckItems(t, r.flocal, file1) + fstest.CheckItems(t, r.Flocal, file1) fs.Stats.ResetCounters() - err := fs.Sync(r.fremote, r.flocal) + err := fs.Sync(r.Fremote, r.Flocal) require.NoError(t, err) // We should have transferred exactly one file. assert.Equal(t, int64(1), fs.Stats.GetTransfers()) - fstest.CheckItems(t, r.fremote, file1) + fstest.CheckItems(t, r.Fremote, file1) // Update mtime, md5sum but not length of file file2 := r.WriteFile("sizeonly", "POTATO", t2) - fstest.CheckItems(t, r.flocal, file2) + fstest.CheckItems(t, r.Flocal, file2) fs.Stats.ResetCounters() - err = fs.Sync(r.fremote, r.flocal) + err = fs.Sync(r.Fremote, r.Flocal) require.NoError(t, err) // We should have transferred no files assert.Equal(t, int64(0), fs.Stats.GetTransfers()) - fstest.CheckItems(t, r.flocal, file2) - fstest.CheckItems(t, r.fremote, file1) + fstest.CheckItems(t, r.Flocal, file2) + fstest.CheckItems(t, r.Fremote, file1) } // Create a file and sync it. Keep the last modified date but change // the size. With --ignore-size we expect nothing to to be // transferred on the second sync. func TestSyncIgnoreSize(t *testing.T) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() fs.Config.IgnoreSize = true defer func() { fs.Config.IgnoreSize = false }() file1 := r.WriteFile("ignore-size", "contents", t1) - fstest.CheckItems(t, r.flocal, file1) + fstest.CheckItems(t, r.Flocal, file1) fs.Stats.ResetCounters() - err := fs.Sync(r.fremote, r.flocal) + err := fs.Sync(r.Fremote, r.Flocal) require.NoError(t, err) // We should have transferred exactly one file. assert.Equal(t, int64(1), fs.Stats.GetTransfers()) - fstest.CheckItems(t, r.fremote, file1) + fstest.CheckItems(t, r.Fremote, file1) // Update size but not date of file file2 := r.WriteFile("ignore-size", "longer contents but same date", t1) - fstest.CheckItems(t, r.flocal, file2) + fstest.CheckItems(t, r.Flocal, file2) fs.Stats.ResetCounters() - err = fs.Sync(r.fremote, r.flocal) + err = fs.Sync(r.Fremote, r.Flocal) require.NoError(t, err) // We should have transferred no files assert.Equal(t, int64(0), fs.Stats.GetTransfers()) - fstest.CheckItems(t, r.flocal, file2) - fstest.CheckItems(t, r.fremote, file1) + fstest.CheckItems(t, r.Flocal, file2) + fstest.CheckItems(t, r.Fremote, file1) } func TestSyncIgnoreTimes(t *testing.T) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() file1 := r.WriteBoth("existing", "potato", t1) - fstest.CheckItems(t, r.fremote, file1) + fstest.CheckItems(t, r.Fremote, file1) fs.Stats.ResetCounters() - err := fs.Sync(r.fremote, r.flocal) + err := fs.Sync(r.Fremote, r.Flocal) require.NoError(t, err) // We should have transferred exactly 0 files because the @@ -267,19 +267,19 @@ func TestSyncIgnoreTimes(t *testing.T) { defer func() { fs.Config.IgnoreTimes = false }() fs.Stats.ResetCounters() - err = fs.Sync(r.fremote, r.flocal) + err = fs.Sync(r.Fremote, r.Flocal) require.NoError(t, err) // We should have transferred exactly one file even though the // files were identical. assert.Equal(t, int64(1), fs.Stats.GetTransfers()) - fstest.CheckItems(t, r.flocal, file1) - fstest.CheckItems(t, r.fremote, file1) + fstest.CheckItems(t, r.Flocal, file1) + fstest.CheckItems(t, r.Fremote, file1) } func TestSyncIgnoreExisting(t *testing.T) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() file1 := r.WriteFile("existing", "potato", t1) @@ -287,54 +287,54 @@ func TestSyncIgnoreExisting(t *testing.T) { defer func() { fs.Config.IgnoreExisting = false }() fs.Stats.ResetCounters() - err := fs.Sync(r.fremote, r.flocal) + err := fs.Sync(r.Fremote, r.Flocal) require.NoError(t, err) - fstest.CheckItems(t, r.flocal, file1) - fstest.CheckItems(t, r.fremote, file1) + fstest.CheckItems(t, r.Flocal, file1) + fstest.CheckItems(t, r.Fremote, file1) // Change everything r.WriteFile("existing", "newpotatoes", t2) fs.Stats.ResetCounters() - err = fs.Sync(r.fremote, r.flocal) + err = fs.Sync(r.Fremote, r.Flocal) require.NoError(t, err) // Items should not change - fstest.CheckItems(t, r.fremote, file1) + fstest.CheckItems(t, r.Fremote, file1) } func TestSyncAfterChangingModtimeOnly(t *testing.T) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() file1 := r.WriteFile("empty space", "", t2) file2 := r.WriteObject("empty space", "", t1) - fstest.CheckItems(t, r.flocal, file1) - fstest.CheckItems(t, r.fremote, file2) + fstest.CheckItems(t, r.Flocal, file1) + fstest.CheckItems(t, r.Fremote, file2) fs.Config.DryRun = true defer func() { fs.Config.DryRun = false }() fs.Stats.ResetCounters() - err := fs.Sync(r.fremote, r.flocal) + err := fs.Sync(r.Fremote, r.Flocal) require.NoError(t, err) - fstest.CheckItems(t, r.flocal, file1) - fstest.CheckItems(t, r.fremote, file2) + fstest.CheckItems(t, r.Flocal, file1) + fstest.CheckItems(t, r.Fremote, file2) fs.Config.DryRun = false fs.Stats.ResetCounters() - err = fs.Sync(r.fremote, r.flocal) + err = fs.Sync(r.Fremote, r.Flocal) require.NoError(t, err) - fstest.CheckItems(t, r.flocal, file1) - fstest.CheckItems(t, r.fremote, file1) + fstest.CheckItems(t, r.Flocal, file1) + fstest.CheckItems(t, r.Fremote, file1) } func TestSyncAfterChangingModtimeOnlyWithNoUpdateModTime(t *testing.T) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() - if r.fremote.Hashes().Count() == 0 { + if r.Fremote.Hashes().Count() == 0 { t.Logf("Can't check this if no hashes supported") return } @@ -347,98 +347,98 @@ func TestSyncAfterChangingModtimeOnlyWithNoUpdateModTime(t *testing.T) { file1 := r.WriteFile("empty space", "", t2) file2 := r.WriteObject("empty space", "", t1) - fstest.CheckItems(t, r.flocal, file1) - fstest.CheckItems(t, r.fremote, file2) + fstest.CheckItems(t, r.Flocal, file1) + fstest.CheckItems(t, r.Fremote, file2) fs.Stats.ResetCounters() - err := fs.Sync(r.fremote, r.flocal) + err := fs.Sync(r.Fremote, r.Flocal) require.NoError(t, err) - fstest.CheckItems(t, r.flocal, file1) - fstest.CheckItems(t, r.fremote, file2) + fstest.CheckItems(t, r.Flocal, file1) + fstest.CheckItems(t, r.Fremote, file2) } func TestSyncDoesntUpdateModtime(t *testing.T) { if fs.Config.ModifyWindow == fs.ModTimeNotSupported { t.Skip("Can't run this test on fs which doesn't support mod time") } - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() file1 := r.WriteFile("foo", "foo", t2) file2 := r.WriteObject("foo", "bar", t1) - fstest.CheckItems(t, r.flocal, file1) - fstest.CheckItems(t, r.fremote, file2) + fstest.CheckItems(t, r.Flocal, file1) + fstest.CheckItems(t, r.Fremote, file2) fs.Stats.ResetCounters() - err := fs.Sync(r.fremote, r.flocal) + err := fs.Sync(r.Fremote, r.Flocal) require.NoError(t, err) - fstest.CheckItems(t, r.flocal, file1) - fstest.CheckItems(t, r.fremote, file1) + fstest.CheckItems(t, r.Flocal, file1) + fstest.CheckItems(t, r.Fremote, file1) // We should have transferred exactly one file, not set the mod time assert.Equal(t, int64(1), fs.Stats.GetTransfers()) } func TestSyncAfterAddingAFile(t *testing.T) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() file1 := r.WriteBoth("empty space", "", t2) file2 := r.WriteFile("potato", "------------------------------------------------------------", t3) - fstest.CheckItems(t, r.flocal, file1, file2) - fstest.CheckItems(t, r.fremote, file1) + fstest.CheckItems(t, r.Flocal, file1, file2) + fstest.CheckItems(t, r.Fremote, file1) fs.Stats.ResetCounters() - err := fs.Sync(r.fremote, r.flocal) + err := fs.Sync(r.Fremote, r.Flocal) require.NoError(t, err) - fstest.CheckItems(t, r.flocal, file1, file2) - fstest.CheckItems(t, r.fremote, file1, file2) + fstest.CheckItems(t, r.Flocal, file1, file2) + fstest.CheckItems(t, r.Fremote, file1, file2) } func TestSyncAfterChangingFilesSizeOnly(t *testing.T) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() file1 := r.WriteObject("potato", "------------------------------------------------------------", t3) file2 := r.WriteFile("potato", "smaller but same date", t3) - fstest.CheckItems(t, r.fremote, file1) - fstest.CheckItems(t, r.flocal, file2) + fstest.CheckItems(t, r.Fremote, file1) + fstest.CheckItems(t, r.Flocal, file2) fs.Stats.ResetCounters() - err := fs.Sync(r.fremote, r.flocal) + err := fs.Sync(r.Fremote, r.Flocal) require.NoError(t, err) - fstest.CheckItems(t, r.flocal, file2) - fstest.CheckItems(t, r.fremote, file2) + fstest.CheckItems(t, r.Flocal, file2) + fstest.CheckItems(t, r.Fremote, file2) } // Sync after changing a file's contents, changing modtime but length // remaining the same func TestSyncAfterChangingContentsOnly(t *testing.T) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() var file1 fstest.Item - if r.fremote.Precision() == fs.ModTimeNotSupported { + if r.Fremote.Precision() == fs.ModTimeNotSupported { t.Logf("ModTimeNotSupported so forcing file to be a different size") file1 = r.WriteObject("potato", "different size to make sure it syncs", t3) } else { file1 = r.WriteObject("potato", "smaller but same date", t3) } file2 := r.WriteFile("potato", "SMALLER BUT SAME DATE", t2) - fstest.CheckItems(t, r.fremote, file1) - fstest.CheckItems(t, r.flocal, file2) + fstest.CheckItems(t, r.Fremote, file1) + fstest.CheckItems(t, r.Flocal, file2) fs.Stats.ResetCounters() - err := fs.Sync(r.fremote, r.flocal) + err := fs.Sync(r.Fremote, r.Flocal) require.NoError(t, err) - fstest.CheckItems(t, r.flocal, file2) - fstest.CheckItems(t, r.fremote, file2) + fstest.CheckItems(t, r.Flocal, file2) + fstest.CheckItems(t, r.Fremote, file2) } // Sync after removing a file and adding a file --dry-run func TestSyncAfterRemovingAFileAndAddingAFileDryRun(t *testing.T) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() file1 := r.WriteFile("potato2", "------------------------------------------------------------", t1) file2 := r.WriteObject("potato", "SMALLER BUT SAME DATE", t2) @@ -446,44 +446,44 @@ func TestSyncAfterRemovingAFileAndAddingAFileDryRun(t *testing.T) { fs.Config.DryRun = true fs.Stats.ResetCounters() - err := fs.Sync(r.fremote, r.flocal) + err := fs.Sync(r.Fremote, r.Flocal) fs.Config.DryRun = false require.NoError(t, err) - fstest.CheckItems(t, r.flocal, file3, file1) - fstest.CheckItems(t, r.fremote, file3, file2) + fstest.CheckItems(t, r.Flocal, file3, file1) + fstest.CheckItems(t, r.Fremote, file3, file2) } // Sync after removing a file and adding a file func TestSyncAfterRemovingAFileAndAddingAFile(t *testing.T) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() file1 := r.WriteFile("potato2", "------------------------------------------------------------", t1) file2 := r.WriteObject("potato", "SMALLER BUT SAME DATE", t2) file3 := r.WriteBoth("empty space", "", t2) - fstest.CheckItems(t, r.fremote, file2, file3) - fstest.CheckItems(t, r.flocal, file1, file3) + fstest.CheckItems(t, r.Fremote, file2, file3) + fstest.CheckItems(t, r.Flocal, file1, file3) fs.Stats.ResetCounters() - err := fs.Sync(r.fremote, r.flocal) + err := fs.Sync(r.Fremote, r.Flocal) require.NoError(t, err) - fstest.CheckItems(t, r.flocal, file1, file3) - fstest.CheckItems(t, r.fremote, file1, file3) + fstest.CheckItems(t, r.Flocal, file1, file3) + fstest.CheckItems(t, r.Fremote, file1, file3) } // Sync after removing a file and adding a file func TestSyncAfterRemovingAFileAndAddingAFileSubDir(t *testing.T) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() file1 := r.WriteFile("a/potato2", "------------------------------------------------------------", t1) file2 := r.WriteObject("b/potato", "SMALLER BUT SAME DATE", t2) file3 := r.WriteBoth("c/non empty space", "AhHa!", t2) - require.NoError(t, fs.Mkdir(r.fremote, "d")) - require.NoError(t, fs.Mkdir(r.fremote, "d/e")) + require.NoError(t, fs.Mkdir(r.Fremote, "d")) + require.NoError(t, fs.Mkdir(r.Fremote, "d/e")) fstest.CheckListingWithPrecision( t, - r.flocal, + r.Flocal, []fstest.Item{ file1, file3, @@ -496,7 +496,7 @@ func TestSyncAfterRemovingAFileAndAddingAFileSubDir(t *testing.T) { ) fstest.CheckListingWithPrecision( t, - r.fremote, + r.Fremote, []fstest.Item{ file2, file3, @@ -511,12 +511,12 @@ func TestSyncAfterRemovingAFileAndAddingAFileSubDir(t *testing.T) { ) fs.Stats.ResetCounters() - err := fs.Sync(r.fremote, r.flocal) + err := fs.Sync(r.Fremote, r.Flocal) require.NoError(t, err) fstest.CheckListingWithPrecision( t, - r.flocal, + r.Flocal, []fstest.Item{ file1, file3, @@ -529,7 +529,7 @@ func TestSyncAfterRemovingAFileAndAddingAFileSubDir(t *testing.T) { ) fstest.CheckListingWithPrecision( t, - r.fremote, + r.Fremote, []fstest.Item{ file1, file3, @@ -544,16 +544,16 @@ func TestSyncAfterRemovingAFileAndAddingAFileSubDir(t *testing.T) { // Sync after removing a file and adding a file with IO Errors func TestSyncAfterRemovingAFileAndAddingAFileSubDirWithErrors(t *testing.T) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() file1 := r.WriteFile("a/potato2", "------------------------------------------------------------", t1) file2 := r.WriteObject("b/potato", "SMALLER BUT SAME DATE", t2) file3 := r.WriteBoth("c/non empty space", "AhHa!", t2) - require.NoError(t, fs.Mkdir(r.fremote, "d")) + require.NoError(t, fs.Mkdir(r.Fremote, "d")) fstest.CheckListingWithPrecision( t, - r.flocal, + r.Flocal, []fstest.Item{ file1, file3, @@ -566,7 +566,7 @@ func TestSyncAfterRemovingAFileAndAddingAFileSubDirWithErrors(t *testing.T) { ) fstest.CheckListingWithPrecision( t, - r.fremote, + r.Fremote, []fstest.Item{ file2, file3, @@ -581,12 +581,12 @@ func TestSyncAfterRemovingAFileAndAddingAFileSubDirWithErrors(t *testing.T) { fs.Stats.ResetCounters() fs.Stats.Error() - err := fs.Sync(r.fremote, r.flocal) + err := fs.Sync(r.Fremote, r.Flocal) assert.Equal(t, fs.ErrorNotDeleting, err) fstest.CheckListingWithPrecision( t, - r.flocal, + r.Flocal, []fstest.Item{ file1, file3, @@ -599,7 +599,7 @@ func TestSyncAfterRemovingAFileAndAddingAFileSubDirWithErrors(t *testing.T) { ) fstest.CheckListingWithPrecision( t, - r.fremote, + r.Fremote, []fstest.Item{ file1, file2, @@ -644,7 +644,7 @@ func TestSyncDeleteBefore(t *testing.T) { // Copy test delete before - shouldn't delete anything func TestCopyDeleteBefore(t *testing.T) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() fs.Config.DeleteMode = fs.DeleteModeBefore @@ -654,26 +654,26 @@ func TestCopyDeleteBefore(t *testing.T) { file1 := r.WriteObject("potato", "hopefully not deleted", t1) file2 := r.WriteFile("potato2", "hopefully copied in", t1) - fstest.CheckItems(t, r.fremote, file1) - fstest.CheckItems(t, r.flocal, file2) + fstest.CheckItems(t, r.Fremote, file1) + fstest.CheckItems(t, r.Flocal, file2) fs.Stats.ResetCounters() - err := fs.CopyDir(r.fremote, r.flocal) + err := fs.CopyDir(r.Fremote, r.Flocal) require.NoError(t, err) - fstest.CheckItems(t, r.fremote, file1, file2) - fstest.CheckItems(t, r.flocal, file2) + fstest.CheckItems(t, r.Fremote, file1, file2) + fstest.CheckItems(t, r.Flocal, file2) } // Test with exclude func TestSyncWithExclude(t *testing.T) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() file1 := r.WriteBoth("potato2", "------------------------------------------------------------", t1) file2 := r.WriteBoth("empty space", "", t2) file3 := r.WriteFile("enormous", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 100 bytes - fstest.CheckItems(t, r.fremote, file1, file2) - fstest.CheckItems(t, r.flocal, file1, file2, file3) + fstest.CheckItems(t, r.Fremote, file1, file2) + fstest.CheckItems(t, r.Flocal, file1, file2, file3) fs.Config.Filter.MaxSize = 40 defer func() { @@ -681,27 +681,27 @@ func TestSyncWithExclude(t *testing.T) { }() fs.Stats.ResetCounters() - err := fs.Sync(r.fremote, r.flocal) + err := fs.Sync(r.Fremote, r.Flocal) require.NoError(t, err) - fstest.CheckItems(t, r.fremote, file2, file1) + fstest.CheckItems(t, r.Fremote, file2, file1) // Now sync the other way round and check enormous doesn't get // deleted as it is excluded from the sync fs.Stats.ResetCounters() - err = fs.Sync(r.flocal, r.fremote) + err = fs.Sync(r.Flocal, r.Fremote) require.NoError(t, err) - fstest.CheckItems(t, r.flocal, file2, file1, file3) + fstest.CheckItems(t, r.Flocal, file2, file1, file3) } // Test with exclude and delete excluded func TestSyncWithExcludeAndDeleteExcluded(t *testing.T) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() file1 := r.WriteBoth("potato2", "------------------------------------------------------------", t1) // 60 bytes file2 := r.WriteBoth("empty space", "", t2) file3 := r.WriteBoth("enormous", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 100 bytes - fstest.CheckItems(t, r.fremote, file1, file2, file3) - fstest.CheckItems(t, r.flocal, file1, file2, file3) + fstest.CheckItems(t, r.Fremote, file1, file2, file3) + fstest.CheckItems(t, r.Flocal, file1, file2, file3) fs.Config.Filter.MaxSize = 40 fs.Config.Filter.DeleteExcluded = true @@ -711,16 +711,16 @@ func TestSyncWithExcludeAndDeleteExcluded(t *testing.T) { }() fs.Stats.ResetCounters() - err := fs.Sync(r.fremote, r.flocal) + err := fs.Sync(r.Fremote, r.Flocal) require.NoError(t, err) - fstest.CheckItems(t, r.fremote, file2) + fstest.CheckItems(t, r.Fremote, file2) // Check sync the other way round to make sure enormous gets // deleted even though it is excluded fs.Stats.ResetCounters() - err = fs.Sync(r.flocal, r.fremote) + err = fs.Sync(r.Flocal, r.Fremote) require.NoError(t, err) - fstest.CheckItems(t, r.flocal, file2) + fstest.CheckItems(t, r.Flocal, file2) } // Test with UpdateOlder set @@ -728,7 +728,7 @@ func TestSyncWithUpdateOlder(t *testing.T) { if fs.Config.ModifyWindow == fs.ModTimeNotSupported { t.Skip("Can't run this test on fs which doesn't support mod time") } - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() t2plus := t2.Add(time.Second / 2) t2minus := t2.Add(time.Second / 2) @@ -737,12 +737,12 @@ func TestSyncWithUpdateOlder(t *testing.T) { threeF := r.WriteFile("three", "three", t2) fourF := r.WriteFile("four", "four", t2) fiveF := r.WriteFile("five", "five", t2) - fstest.CheckItems(t, r.flocal, oneF, twoF, threeF, fourF, fiveF) + fstest.CheckItems(t, r.Flocal, oneF, twoF, threeF, fourF, fiveF) oneO := r.WriteObject("one", "ONE", t2) twoO := r.WriteObject("two", "TWO", t2) threeO := r.WriteObject("three", "THREE", t2plus) fourO := r.WriteObject("four", "FOURFOUR", t2minus) - fstest.CheckItems(t, r.fremote, oneO, twoO, threeO, fourO) + fstest.CheckItems(t, r.Fremote, oneO, twoO, threeO, fourO) fs.Config.UpdateOlder = true oldModifyWindow := fs.Config.ModifyWindow @@ -753,14 +753,14 @@ func TestSyncWithUpdateOlder(t *testing.T) { }() fs.Stats.ResetCounters() - err := fs.Sync(r.fremote, r.flocal) + err := fs.Sync(r.Fremote, r.Flocal) require.NoError(t, err) - fstest.CheckItems(t, r.fremote, oneO, twoF, threeO, fourF, fiveF) + fstest.CheckItems(t, r.Fremote, oneO, twoF, threeO, fourF, fiveF) } // Test with TrackRenames set func TestSyncWithTrackRenames(t *testing.T) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() fs.Config.TrackRenames = true @@ -769,26 +769,26 @@ func TestSyncWithTrackRenames(t *testing.T) { }() - haveHash := r.fremote.Hashes().Overlap(r.flocal.Hashes()).GetOne() != fs.HashNone - canTrackRenames := haveHash && fs.CanServerSideMove(r.fremote) + haveHash := r.Fremote.Hashes().Overlap(r.Flocal.Hashes()).GetOne() != fs.HashNone + canTrackRenames := haveHash && fs.CanServerSideMove(r.Fremote) t.Logf("Can track renames: %v", canTrackRenames) f1 := r.WriteFile("potato", "Potato Content", t1) f2 := r.WriteFile("yam", "Yam Content", t2) fs.Stats.ResetCounters() - require.NoError(t, fs.Sync(r.fremote, r.flocal)) + require.NoError(t, fs.Sync(r.Fremote, r.Flocal)) - fstest.CheckItems(t, r.fremote, f1, f2) - fstest.CheckItems(t, r.flocal, f1, f2) + fstest.CheckItems(t, r.Fremote, f1, f2) + fstest.CheckItems(t, r.Flocal, f1, f2) // Now rename locally. f2 = r.RenameFile(f2, "yaml") fs.Stats.ResetCounters() - require.NoError(t, fs.Sync(r.fremote, r.flocal)) + require.NoError(t, fs.Sync(r.Fremote, r.Flocal)) - fstest.CheckItems(t, r.fremote, f1, f2) + fstest.CheckItems(t, r.Fremote, f1, f2) if canTrackRenames { assert.Equal(t, fs.Stats.GetTransfers(), int64(0)) @@ -798,8 +798,8 @@ func TestSyncWithTrackRenames(t *testing.T) { } // Test a server side move if possible, or the backup path if not -func testServerSideMove(t *testing.T, r *Run, withFilter bool) { - fremoteMove, _, finaliseMove, err := fstest.RandomRemote(*fstest.RemoteName, *fstest.SubDir) +func testServerSideMove(t *testing.T, r *fstest.Run, withFilter bool) { + FremoteMove, _, finaliseMove, err := fstest.RandomRemote(*fstest.RemoteName, *fstest.SubDir) require.NoError(t, err) defer finaliseMove() @@ -807,56 +807,56 @@ func testServerSideMove(t *testing.T, r *Run, withFilter bool) { file2 := r.WriteBoth("empty space", "", t2) file3u := r.WriteBoth("potato3", "------------------------------------------------------------ UPDATED", t2) - fstest.CheckItems(t, r.fremote, file2, file1, file3u) + fstest.CheckItems(t, r.Fremote, file2, file1, file3u) - t.Logf("Server side move (if possible) %v -> %v", r.fremote, fremoteMove) + t.Logf("Server side move (if possible) %v -> %v", r.Fremote, FremoteMove) // Write just one file in the new remote - r.WriteObjectTo(fremoteMove, "empty space", "", t2, false) - file3 := r.WriteObjectTo(fremoteMove, "potato3", "------------------------------------------------------------", t1, false) - fstest.CheckItems(t, fremoteMove, file2, file3) + r.WriteObjectTo(FremoteMove, "empty space", "", t2, false) + file3 := r.WriteObjectTo(FremoteMove, "potato3", "------------------------------------------------------------", t1, false) + fstest.CheckItems(t, FremoteMove, file2, file3) // Do server side move fs.Stats.ResetCounters() - err = fs.MoveDir(fremoteMove, r.fremote) + err = fs.MoveDir(FremoteMove, r.Fremote) require.NoError(t, err) if withFilter { - fstest.CheckItems(t, r.fremote, file2) + fstest.CheckItems(t, r.Fremote, file2) } else { - fstest.CheckItems(t, r.fremote) + fstest.CheckItems(t, r.Fremote) } - fstest.CheckItems(t, fremoteMove, file2, file1, file3u) + fstest.CheckItems(t, FremoteMove, file2, file1, file3u) // Create a new empty remote for stuff to be moved into - fremoteMove2, _, finaliseMove2, err := fstest.RandomRemote(*fstest.RemoteName, *fstest.SubDir) + FremoteMove2, _, finaliseMove2, err := fstest.RandomRemote(*fstest.RemoteName, *fstest.SubDir) require.NoError(t, err) defer finaliseMove2() // Move it back to a new empty remote, dst does not exist this time fs.Stats.ResetCounters() - err = fs.MoveDir(fremoteMove2, fremoteMove) + err = fs.MoveDir(FremoteMove2, FremoteMove) require.NoError(t, err) if withFilter { - fstest.CheckItems(t, fremoteMove2, file1, file3u) - fstest.CheckItems(t, fremoteMove, file2) + fstest.CheckItems(t, FremoteMove2, file1, file3u) + fstest.CheckItems(t, FremoteMove, file2) } else { - fstest.CheckItems(t, fremoteMove2, file2, file1, file3u) - fstest.CheckItems(t, fremoteMove) + fstest.CheckItems(t, FremoteMove2, file2, file1, file3u) + fstest.CheckItems(t, FremoteMove) } } // Test a server side move if possible, or the backup path if not func TestServerSideMove(t *testing.T) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() testServerSideMove(t, r, false) } // Test a server side move if possible, or the backup path if not func TestServerSideMoveWithFilter(t *testing.T) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() fs.Config.Filter.MinSize = 40 @@ -869,22 +869,22 @@ func TestServerSideMoveWithFilter(t *testing.T) { // Test a server side move with overlap func TestServerSideMoveOverlap(t *testing.T) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() - if r.fremote.Features().DirMove != nil { + if r.Fremote.Features().DirMove != nil { t.Skip("Skipping test as remote supports DirMove") } - subRemoteName := r.fremoteName + "/rclone-move-test" - fremoteMove, err := fs.NewFs(subRemoteName) + subRemoteName := r.FremoteName + "/rclone-move-test" + FremoteMove, err := fs.NewFs(subRemoteName) require.NoError(t, err) file1 := r.WriteObject("potato2", "------------------------------------------------------------", t1) - fstest.CheckItems(t, r.fremote, file1) + fstest.CheckItems(t, r.Fremote, file1) // Subdir move with no filters should return ErrorCantMoveOverlapping - err = fs.MoveDir(fremoteMove, r.fremote) + err = fs.MoveDir(FremoteMove, r.Fremote) assert.EqualError(t, err, fs.ErrorCantMoveOverlapping.Error()) // Now try with a filter which should also fail with ErrorCantMoveOverlapping @@ -892,21 +892,21 @@ func TestServerSideMoveOverlap(t *testing.T) { defer func() { fs.Config.Filter.MinSize = -1 }() - err = fs.MoveDir(fremoteMove, r.fremote) + err = fs.MoveDir(FremoteMove, r.Fremote) assert.EqualError(t, err, fs.ErrorCantMoveOverlapping.Error()) } // Test with BackupDir set func testSyncBackupDir(t *testing.T, suffix string) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() - if !fs.CanServerSideMove(r.fremote) { + if !fs.CanServerSideMove(r.Fremote) { t.Skip("Skipping test as remote does not support server side move") } - r.Mkdir(r.fremote) + r.Mkdir(r.Fremote) - fs.Config.BackupDir = r.fremoteName + "/backup" + fs.Config.BackupDir = r.FremoteName + "/backup" fs.Config.Suffix = suffix defer func() { fs.Config.BackupDir = "" @@ -921,14 +921,14 @@ func testSyncBackupDir(t *testing.T, suffix string) { file2a := r.WriteFile("two", "two", t1) file1a := r.WriteFile("one", "oneA", t2) - fstest.CheckItems(t, r.fremote, file1, file2, file3) - fstest.CheckItems(t, r.flocal, file1a, file2a) + fstest.CheckItems(t, r.Fremote, file1, file2, file3) + fstest.CheckItems(t, r.Flocal, file1a, file2a) - fdst, err := fs.NewFs(r.fremoteName + "/dst") + fdst, err := fs.NewFs(r.FremoteName + "/dst") require.NoError(t, err) fs.Stats.ResetCounters() - err = fs.Sync(fdst, r.flocal) + err = fs.Sync(fdst, r.Flocal) require.NoError(t, err) // one should be moved to the backup dir and the new one installed @@ -938,18 +938,18 @@ func testSyncBackupDir(t *testing.T, suffix string) { // three should be moved to the backup dir file3.Path = "backup/three" + suffix - fstest.CheckItems(t, r.fremote, file1, file2, file3, file1a) + fstest.CheckItems(t, r.Fremote, file1, file2, file3, file1a) // Now check what happens if we do it again // Restore a different three and update one in the source file3a := r.WriteObject("dst/three", "threeA", t2) file1b := r.WriteFile("one", "oneBB", t3) - fstest.CheckItems(t, r.fremote, file1, file2, file3, file1a, file3a) + fstest.CheckItems(t, r.Fremote, file1, file2, file3, file1a, file3a) // This should delete three and overwrite one again, checking // the files got overwritten correctly in backup-dir fs.Stats.ResetCounters() - err = fs.Sync(fdst, r.flocal) + err = fs.Sync(fdst, r.Flocal) require.NoError(t, err) // one should be moved to the backup dir and the new one installed @@ -959,7 +959,7 @@ func testSyncBackupDir(t *testing.T, suffix string) { // three should be moved to the backup dir file3a.Path = "backup/three" + suffix - fstest.CheckItems(t, r.fremote, file1b, file2, file3a, file1a) + fstest.CheckItems(t, r.Fremote, file1b, file2, file3a, file1a) } func TestSyncBackupDir(t *testing.T) { testSyncBackupDir(t, "") } func TestSyncBackupDirWithSuffix(t *testing.T) { testSyncBackupDir(t, ".bak") } @@ -970,7 +970,7 @@ func TestSyncUTFNorm(t *testing.T) { t.Skip("Can't test UTF normalization on OS X") } - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() // Two strings with different unicode normalization (from OS X) @@ -980,26 +980,26 @@ func TestSyncUTFNorm(t *testing.T) { assert.Equal(t, norm.NFC.String(Encoding1), norm.NFC.String(Encoding2)) file1 := r.WriteFile(Encoding1, "This is a test", t1) - fstest.CheckItems(t, r.flocal, file1) + fstest.CheckItems(t, r.Flocal, file1) file2 := r.WriteObject(Encoding2, "This is a old test", t2) - fstest.CheckItems(t, r.fremote, file2) + fstest.CheckItems(t, r.Fremote, file2) fs.Stats.ResetCounters() - err := fs.Sync(r.fremote, r.flocal) + err := fs.Sync(r.Fremote, r.Flocal) require.NoError(t, err) // We should have transferred exactly one file, but kept the // normalized state of the file. assert.Equal(t, int64(1), fs.Stats.GetTransfers()) - fstest.CheckItems(t, r.flocal, file1) + fstest.CheckItems(t, r.Flocal, file1) file1.Path = file2.Path - fstest.CheckItems(t, r.fremote, file1) + fstest.CheckItems(t, r.Fremote, file1) } // Test --immutable func TestSyncImmutable(t *testing.T) { - r := NewRun(t) + r := fstest.NewRun(t) defer r.Finalise() fs.Config.Immutable = true @@ -1007,25 +1007,25 @@ func TestSyncImmutable(t *testing.T) { // Create file on source file1 := r.WriteFile("existing", "potato", t1) - fstest.CheckItems(t, r.flocal, file1) - fstest.CheckItems(t, r.fremote) + fstest.CheckItems(t, r.Flocal, file1) + fstest.CheckItems(t, r.Fremote) // Should succeed fs.Stats.ResetCounters() - err := fs.Sync(r.fremote, r.flocal) + err := fs.Sync(r.Fremote, r.Flocal) require.NoError(t, err) - fstest.CheckItems(t, r.flocal, file1) - fstest.CheckItems(t, r.fremote, file1) + fstest.CheckItems(t, r.Flocal, file1) + fstest.CheckItems(t, r.Fremote, file1) // Modify file data and timestamp on source file2 := r.WriteFile("existing", "tomatoes", t2) - fstest.CheckItems(t, r.flocal, file2) - fstest.CheckItems(t, r.fremote, file1) + fstest.CheckItems(t, r.Flocal, file2) + fstest.CheckItems(t, r.Fremote, file1) // Should fail with ErrorImmutableModified and not modify local or remote files fs.Stats.ResetCounters() - err = fs.Sync(r.fremote, r.flocal) + err = fs.Sync(r.Fremote, r.Flocal) assert.EqualError(t, err, fs.ErrorImmutableModified.Error()) - fstest.CheckItems(t, r.flocal, file2) - fstest.CheckItems(t, r.fremote, file1) + fstest.CheckItems(t, r.Flocal, file2) + fstest.CheckItems(t, r.Fremote, file1) } diff --git a/fstest/run.go b/fstest/run.go new file mode 100644 index 000000000..1285ad888 --- /dev/null +++ b/fstest/run.go @@ -0,0 +1,288 @@ +/* + +This provides Run for use in creating test suites + +To use this declare a TestMain + +// TestMain drives the tests +func TestMain(m *testing.M) { + fstest.TestMain(m) +} + +And then make and destroy a Run in each test + +func TestMkdir(t *testing.T) { + r := fstest.NewRun(t) + defer r.Finalise() + // test stuff +} + +This will make r.Fremote and r.Flocal for a remote remote and a local +remote. The remote is determined by the -remote flag passed in. + +*/ + +package fstest + +import ( + "bytes" + "flag" + "io/ioutil" + "log" + "os" + "path" + "path/filepath" + "sort" + "testing" + "time" + + "github.com/ncw/rclone/fs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Run holds the remotes for a test run +type Run struct { + LocalName string + Flocal fs.Fs + Fremote fs.Fs + FremoteName string + cleanRemote func() + mkdir map[string]bool // whether the remote has been made yet for the fs name + Logf, Fatalf func(text string, args ...interface{}) +} + +// TestMain drives the tests +func TestMain(m *testing.M) { + flag.Parse() + if !*Individual { + oneRun = newRun() + } + rc := m.Run() + if !*Individual { + oneRun.Finalise() + } + os.Exit(rc) +} + +// oneRun holds the master run data if individual is not set +var oneRun *Run + +// newRun initialise the remote and local for testing and returns a +// run object. +// +// r.Flocal is an empty local Fs +// r.Fremote is an empty remote Fs +// +// Finalise() will tidy them away when done. +func newRun() *Run { + r := &Run{ + Logf: log.Printf, + Fatalf: log.Fatalf, + mkdir: make(map[string]bool), + } + + Initialise() + + var err error + r.Fremote, r.FremoteName, r.cleanRemote, err = RandomRemote(*RemoteName, *SubDir) + if err != nil { + r.Fatalf("Failed to open remote %q: %v", *RemoteName, err) + } + + r.LocalName, err = ioutil.TempDir("", "rclone") + if err != nil { + r.Fatalf("Failed to create temp dir: %v", err) + } + r.LocalName = filepath.ToSlash(r.LocalName) + r.Flocal, err = fs.NewFs(r.LocalName) + if err != nil { + r.Fatalf("Failed to make %q: %v", r.LocalName, err) + } + fs.CalculateModifyWindow(r.Fremote, r.Flocal) + return r +} + +// dirsToRemove sorts by string length +type dirsToRemove []string + +func (d dirsToRemove) Len() int { return len(d) } +func (d dirsToRemove) Swap(i, j int) { d[i], d[j] = d[j], d[i] } +func (d dirsToRemove) Less(i, j int) bool { return len(d[i]) > len(d[j]) } + +// NewRun initialise the remote and local for testing and returns a +// run object. Call this from the tests. +// +// r.Flocal is an empty local Fs +// r.Fremote is an empty remote Fs +// +// Finalise() will tidy them away when done. +func NewRun(t *testing.T) *Run { + var r *Run + if *Individual { + r = newRun() + } else { + // If not individual, use the global one with the clean method overridden + r = new(Run) + *r = *oneRun + r.cleanRemote = func() { + var toDelete dirsToRemove + require.NoError(t, fs.Walk(r.Fremote, "", true, -1, func(dirPath string, entries fs.DirEntries, err error) error { + if err != nil { + if err == fs.ErrorDirNotFound { + return nil + } + t.Fatalf("Error listing: %v", err) + } + for _, entry := range entries { + switch x := entry.(type) { + case fs.Object: + err = x.Remove() + if err != nil { + t.Errorf("Error removing file %q: %v", x.Remote(), err) + } + case fs.Directory: + toDelete = append(toDelete, x.Remote()) + } + } + return nil + })) + sort.Sort(toDelete) + for _, dir := range toDelete { + err := r.Fremote.Rmdir(dir) + if err != nil { + t.Errorf("Error removing dir %q: %v", dir, err) + } + } + // Check remote is empty + CheckItems(t, r.Fremote) + } + } + r.Logf = t.Logf + r.Fatalf = t.Fatalf + r.Logf("Remote %q, Local %q, Modify Window %q", r.Fremote, r.Flocal, fs.Config.ModifyWindow) + return r +} + +// RenameFile renames a file in local +func (r *Run) RenameFile(item Item, newpath string) Item { + oldFilepath := path.Join(r.LocalName, item.Path) + newFilepath := path.Join(r.LocalName, newpath) + if err := os.Rename(oldFilepath, newFilepath); err != nil { + r.Fatalf("Failed to rename file from %q to %q: %v", item.Path, newpath, err) + } + + item.Path = newpath + + return item +} + +// WriteFile writes a file to local +func (r *Run) WriteFile(filePath, content string, t time.Time) Item { + item := NewItem(filePath, content, t) + // FIXME make directories? + filePath = path.Join(r.LocalName, filePath) + dirPath := path.Dir(filePath) + err := os.MkdirAll(dirPath, 0770) + if err != nil { + r.Fatalf("Failed to make directories %q: %v", dirPath, err) + } + err = ioutil.WriteFile(filePath, []byte(content), 0600) + if err != nil { + r.Fatalf("Failed to write file %q: %v", filePath, err) + } + err = os.Chtimes(filePath, t, t) + if err != nil { + r.Fatalf("Failed to chtimes file %q: %v", filePath, err) + } + return item +} + +// ForceMkdir creates the remote +func (r *Run) ForceMkdir(f fs.Fs) { + err := f.Mkdir("") + if err != nil { + r.Fatalf("Failed to mkdir %q: %v", f, err) + } + r.mkdir[f.String()] = true +} + +// Mkdir creates the remote if it hasn't been created already +func (r *Run) Mkdir(f fs.Fs) { + if !r.mkdir[f.String()] { + r.ForceMkdir(f) + } +} + +// WriteObjectTo writes an object to the fs, remote passed in +func (r *Run) WriteObjectTo(f fs.Fs, remote, content string, modTime time.Time, useUnchecked bool) Item { + put := f.Put + if useUnchecked { + put = f.Features().PutUnchecked + if put == nil { + r.Fatalf("Fs doesn't support PutUnchecked") + } + } + r.Mkdir(f) + const maxTries = 10 + for tries := 1; ; tries++ { + in := bytes.NewBufferString(content) + objinfo := fs.NewStaticObjectInfo(remote, modTime, int64(len(content)), true, nil, nil) + _, err := put(in, objinfo) + if err == nil { + break + } + // Retry if err returned a retry error + if fs.IsRetryError(err) && tries < maxTries { + r.Logf("Retry Put of %q to %v: %d/%d (%v)", remote, f, tries, maxTries, err) + time.Sleep(2 * time.Second) + continue + } + r.Fatalf("Failed to put %q to %q: %v", remote, f, err) + } + return NewItem(remote, content, modTime) +} + +// WriteObject writes an object to the remote +func (r *Run) WriteObject(remote, content string, modTime time.Time) Item { + return r.WriteObjectTo(r.Fremote, remote, content, modTime, false) +} + +// WriteUncheckedObject writes an object to the remote not checking for duplicates +func (r *Run) WriteUncheckedObject(remote, content string, modTime time.Time) Item { + return r.WriteObjectTo(r.Fremote, remote, content, modTime, true) +} + +// WriteBoth calls WriteObject and WriteFile with the same arguments +func (r *Run) WriteBoth(remote, content string, modTime time.Time) Item { + r.WriteFile(remote, content, modTime) + return r.WriteObject(remote, content, modTime) +} + +// CheckWithDuplicates does a test but allows duplicates +func (r *Run) CheckWithDuplicates(t *testing.T, items ...Item) { + objects, size, err := fs.Count(r.Fremote) + require.NoError(t, err) + assert.Equal(t, int64(len(items)), objects) + wantSize := int64(0) + for _, item := range items { + wantSize += item.Size + } + assert.Equal(t, wantSize, size) +} + +// Clean the temporary directory +func (r *Run) cleanTempDir() { + err := os.RemoveAll(r.LocalName) + if err != nil { + r.Logf("Failed to clean temporary directory %q: %v", r.LocalName, err) + } +} + +// Finalise cleans the remote and local +func (r *Run) Finalise() { + // r.Logf("Cleaning remote %q", r.Fremote) + r.cleanRemote() + // r.Logf("Cleaning local %q", r.LocalName) + r.cleanTempDir() +}