diff --git a/backend/azureblob/azureblob.go b/backend/azureblob/azureblob.go index 44345058d..8634d33f9 100644 --- a/backend/azureblob/azureblob.go +++ b/backend/azureblob/azureblob.go @@ -1453,7 +1453,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op blob := o.getBlobReference() httpHeaders := azblob.BlobHTTPHeaders{} - httpHeaders.ContentType = fs.MimeType(ctx, o) + httpHeaders.ContentType = fs.MimeType(ctx, src) // Compute the Content-MD5 of the file, for multiparts uploads it // will be set in PutBlockList API call using the 'x-ms-blob-content-md5' header // Note: If multipart, an MD5 checksum will also be computed for each uploaded block diff --git a/backend/memory/memory.go b/backend/memory/memory.go index cda1c0c28..0ff851d45 100644 --- a/backend/memory/memory.go +++ b/backend/memory/memory.go @@ -592,7 +592,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op data: data, hash: "", modTime: src.ModTime(ctx), - mimeType: fs.MimeType(ctx, o), + mimeType: fs.MimeType(ctx, src), } buckets.updateObjectData(bucket, bucketPath, o.od) return nil diff --git a/backend/pcloud/pcloud.go b/backend/pcloud/pcloud.go index 0e564e038..58734e18e 100644 --- a/backend/pcloud/pcloud.go +++ b/backend/pcloud/pcloud.go @@ -1117,7 +1117,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op Method: "PUT", Path: "/uploadfile", Body: in, - ContentType: fs.MimeType(ctx, o), + ContentType: fs.MimeType(ctx, src), ContentLength: &size, Parameters: url.Values{}, TransferEncoding: []string{"identity"}, // pcloud doesn't like chunked encoding diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go index 9a13c43fa..622825afd 100644 --- a/fstest/fstests/fstests.go +++ b/fstest/fstests/fstests.go @@ -153,13 +153,32 @@ func retry(t *testing.T, what string, f func() error) { require.NoError(t, err, what) } -// testPut puts file with random contents to the remote -func testPut(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item) (string, fs.Object) { - return PutTestContents(ctx, t, f, file, random.String(100), true) +// An fs.ObjectInfo that can override mime type +type objectInfoWithMimeType struct { + fs.ObjectInfo + mimeType string } -// PutTestContents puts file with given contents to the remote and checks it but unlike TestPutLarge doesn't remove -func PutTestContents(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item, contents string, check bool) (string, fs.Object) { +// Return a wrapped fs.ObjectInfo which returns the mime type given +func overrideMimeType(o fs.ObjectInfo, mimeType string) fs.ObjectInfo { + return &objectInfoWithMimeType{ + ObjectInfo: o, + mimeType: mimeType, + } +} + +// MimeType that was overridden +func (o *objectInfoWithMimeType) MimeType(ctx context.Context) string { + return o.mimeType +} + +// check interface +var _ fs.MimeTyper = (*objectInfoWithMimeType)(nil) + +// putTestContentsMimeType puts file with given contents to the remote and checks it but unlike TestPutLarge doesn't remove +// +// it uploads the object with the mimeType passed in if set +func putTestContentsMimeType(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item, contents string, check bool, mimeType string) (string, fs.Object) { var ( err error obj fs.Object @@ -172,6 +191,9 @@ func PutTestContents(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.It file.Size = int64(buf.Len()) obji := object.NewStaticObjectInfo(file.Path, file.ModTime, file.Size, true, nil, nil) + if mimeType != "" { + obji = overrideMimeType(obji, mimeType) + } obj, err = f.Put(ctx, in, obji) return err }) @@ -185,6 +207,21 @@ func PutTestContents(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.It return contents, obj } +// PutTestContents puts file with given contents to the remote and checks it but unlike TestPutLarge doesn't remove +func PutTestContents(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item, contents string, check bool) (string, fs.Object) { + return putTestContentsMimeType(ctx, t, f, file, contents, check, "") +} + +// testPut puts file with random contents to the remote +func testPut(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item) (string, fs.Object) { + return PutTestContents(ctx, t, f, file, random.String(100), true) +} + +// testPutMimeType puts file with random contents to the remote and the mime type given +func testPutMimeType(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item, mimeType string) (string, fs.Object) { + return putTestContentsMimeType(ctx, t, f, file, random.String(100), true, mimeType) +} + // TestPutLarge puts file to the remote, checks it and removes it on success. func TestPutLarge(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item) { var ( @@ -279,7 +316,7 @@ func stringsContains(x string, ss []string) bool { // - these are inside the "FsPutFiles" test. func Run(t *testing.T, opt *Opt) { var ( - remote fs.Fs + f fs.Fs remoteName = opt.RemoteName subRemoteName string subRemoteLeaf string @@ -288,6 +325,7 @@ func Run(t *testing.T, opt *Opt) { Path: "file name.txt", } file1Contents string + file1MimeType = "text/csv" file2 = fstest.Item{ ModTime: fstest.Time("2001-02-03T04:05:10.123123123Z"), Path: `hello? sausage/êé/Hello, 世界/ " ' @ < > & ? + ≠/z.txt`, @@ -305,7 +343,7 @@ func Run(t *testing.T, opt *Opt) { // Skip the test if the remote isn't configured skipIfNotOk := func(t *testing.T) { - if remote == nil { + if f == nil { t.Skipf("WARN: %q not configured", remoteName) } } @@ -314,7 +352,7 @@ func Run(t *testing.T, opt *Opt) { // flag, returning a function to restore its value skipIfNotListR := func(t *testing.T) func() { skipIfNotOk(t) - if remote.Features().ListR == nil { + if f.Features().ListR == nil { t.Skip("FS has no ListR interface") } previous := ci.UseListR @@ -327,8 +365,8 @@ func Run(t *testing.T, opt *Opt) { // Skip if remote is not SetTier and GetTier capable skipIfNotSetTier := func(t *testing.T) { skipIfNotOk(t) - if remote.Features().SetTier == false || - remote.Features().GetTier == false { + if f.Features().SetTier == false || + f.Features().GetTier == false { t.Skip("FS has no SetTier & GetTier interfaces") } } @@ -374,7 +412,7 @@ func Run(t *testing.T, opt *Opt) { // remote - the result of fs.NewFs(TestRemote:subRemoteName) subRemoteName, subRemoteLeaf, err = fstest.RandomRemoteName(remoteName) require.NoError(t, err) - remote, err = fs.NewFs(context.Background(), subRemoteName) + f, err = fs.NewFs(context.Background(), subRemoteName) if err == fs.ErrorNotFoundInConfigFile { t.Logf("Didn't find %q in config file - skipping tests", remoteName) return @@ -390,7 +428,7 @@ func Run(t *testing.T, opt *Opt) { if opt.SkipFsCheckWrap { t.Skip("Skipping FsCheckWrap on this Fs") } - ft := new(fs.Features).Fill(ctx, remote) + ft := new(fs.Features).Fill(ctx, f) if ft.UnWrap == nil { t.Skip("Not a wrapping Fs") } @@ -418,7 +456,7 @@ func Run(t *testing.T, opt *Opt) { // Check to see if Fs advertises commands and they work and have docs t.Run("FsCommand", func(t *testing.T) { skipIfNotOk(t) - doCommand := remote.Features().Command + doCommand := f.Features().Command if doCommand == nil { t.Skip("No commands in this remote") } @@ -434,29 +472,29 @@ func Run(t *testing.T, opt *Opt) { // TestFsRmdirNotFound tests deleting a non existent directory t.Run("FsRmdirNotFound", func(t *testing.T) { skipIfNotOk(t) - if isBucketBasedButNotRoot(remote) { + if isBucketBasedButNotRoot(f) { t.Skip("Skipping test as non root bucket based remote") } - err := remote.Rmdir(ctx, "") + err := f.Rmdir(ctx, "") assert.Error(t, err, "Expecting error on Rmdir non existent") }) // Make the directory - err = remote.Mkdir(ctx, "") + err = f.Mkdir(ctx, "") require.NoError(t, err) - fstest.CheckListing(t, remote, []fstest.Item{}) + fstest.CheckListing(t, f, []fstest.Item{}) // TestFsString tests the String method t.Run("FsString", func(t *testing.T) { skipIfNotOk(t) - str := remote.String() + str := f.String() require.NotEqual(t, "", str) }) // TestFsName tests the Name method t.Run("FsName", func(t *testing.T) { skipIfNotOk(t) - got := remote.Name() + got := f.Name() want := remoteName[:strings.LastIndex(remoteName, ":")+1] if isLocalRemote { want = "local:" @@ -467,8 +505,8 @@ func Run(t *testing.T, opt *Opt) { // TestFsRoot tests the Root method t.Run("FsRoot", func(t *testing.T) { skipIfNotOk(t) - name := remote.Name() + ":" - root := remote.Root() + name := f.Name() + ":" + root := f.Root() if isLocalRemote { // only check last path element on local require.Equal(t, filepath.Base(subRemoteName), filepath.Base(root)) @@ -480,7 +518,7 @@ func Run(t *testing.T, opt *Opt) { // TestFsRmdirEmpty tests deleting an empty directory t.Run("FsRmdirEmpty", func(t *testing.T) { skipIfNotOk(t) - err := remote.Rmdir(ctx, "") + err := f.Rmdir(ctx, "") require.NoError(t, err) }) @@ -490,41 +528,41 @@ func Run(t *testing.T, opt *Opt) { t.Run("FsMkdir", func(t *testing.T) { skipIfNotOk(t) - err := remote.Mkdir(ctx, "") + err := f.Mkdir(ctx, "") require.NoError(t, err) - fstest.CheckListing(t, remote, []fstest.Item{}) + fstest.CheckListing(t, f, []fstest.Item{}) - err = remote.Mkdir(ctx, "") + err = f.Mkdir(ctx, "") require.NoError(t, err) // TestFsMkdirRmdirSubdir tests making and removing a sub directory t.Run("FsMkdirRmdirSubdir", func(t *testing.T) { skipIfNotOk(t) dir := "dir/subdir" - err := operations.Mkdir(ctx, remote, dir) + err := operations.Mkdir(ctx, f, dir) require.NoError(t, err) - fstest.CheckListingWithPrecision(t, remote, []fstest.Item{}, []string{"dir", "dir/subdir"}, fs.GetModifyWindow(ctx, remote)) + fstest.CheckListingWithPrecision(t, f, []fstest.Item{}, []string{"dir", "dir/subdir"}, fs.GetModifyWindow(ctx, f)) - err = operations.Rmdir(ctx, remote, dir) + err = operations.Rmdir(ctx, f, dir) require.NoError(t, err) - fstest.CheckListingWithPrecision(t, remote, []fstest.Item{}, []string{"dir"}, fs.GetModifyWindow(ctx, remote)) + fstest.CheckListingWithPrecision(t, f, []fstest.Item{}, []string{"dir"}, fs.GetModifyWindow(ctx, f)) - err = operations.Rmdir(ctx, remote, "dir") + err = operations.Rmdir(ctx, f, "dir") require.NoError(t, err) - fstest.CheckListingWithPrecision(t, remote, []fstest.Item{}, []string{}, fs.GetModifyWindow(ctx, remote)) + fstest.CheckListingWithPrecision(t, f, []fstest.Item{}, []string{}, fs.GetModifyWindow(ctx, f)) }) // TestFsListEmpty tests listing an empty directory t.Run("FsListEmpty", func(t *testing.T) { skipIfNotOk(t) - fstest.CheckListing(t, remote, []fstest.Item{}) + fstest.CheckListing(t, f, []fstest.Item{}) }) // TestFsListDirEmpty tests listing the directories from an empty directory TestFsListDirEmpty := func(t *testing.T) { skipIfNotOk(t) - objs, dirs, err := walk.GetAll(ctx, remote, "", true, 1) - if !remote.Features().CanHaveEmptyDirectories { + objs, dirs, err := walk.GetAll(ctx, f, "", true, 1) + if !f.Features().CanHaveEmptyDirectories { if err != fs.ErrorDirNotFound { require.NoError(t, err) } @@ -545,8 +583,8 @@ func Run(t *testing.T, opt *Opt) { // TestFsListDirNotFound tests listing the directories from an empty directory TestFsListDirNotFound := func(t *testing.T) { skipIfNotOk(t) - objs, dirs, err := walk.GetAll(ctx, remote, "does not exist", true, 1) - if !remote.Features().CanHaveEmptyDirectories { + objs, dirs, err := walk.GetAll(ctx, f, "does not exist", true, 1) + if !f.Features().CanHaveEmptyDirectories { if err != fs.ErrorDirNotFound { assert.NoError(t, err) assert.Equal(t, 0, len(objs)+len(dirs)) @@ -570,7 +608,7 @@ func Run(t *testing.T, opt *Opt) { skipIfNotOk(t) // check no files or dirs as pre-requisite - fstest.CheckListingWithPrecision(t, remote, []fstest.Item{}, []string{}, fs.GetModifyWindow(ctx, remote)) + fstest.CheckListingWithPrecision(t, f, []fstest.Item{}, []string{}, fs.GetModifyWindow(ctx, f)) for _, test := range []struct { name string @@ -604,16 +642,16 @@ func Run(t *testing.T, opt *Opt) { fileName := encoder.Standard.Encode(test.path) dirName := fileName t.Logf("testing %q", fileName) - assert.NoError(t, remote.Mkdir(ctx, dirName)) + assert.NoError(t, f.Mkdir(ctx, dirName)) file := fstest.Item{ ModTime: time.Now(), Path: dirName + "/" + fileName, // test creating a file and dir with that name } - _, o := testPut(context.Background(), t, remote, &file) - fstest.CheckListingWithPrecision(t, remote, []fstest.Item{file}, []string{dirName}, fs.GetModifyWindow(ctx, remote)) + _, o := testPut(context.Background(), t, f, &file) + fstest.CheckListingWithPrecision(t, f, []fstest.Item{file}, []string{dirName}, fs.GetModifyWindow(ctx, f)) assert.NoError(t, o.Remove(ctx)) - assert.NoError(t, remote.Rmdir(ctx, dirName)) - fstest.CheckListingWithPrecision(t, remote, []fstest.Item{}, []string{}, fs.GetModifyWindow(ctx, remote)) + assert.NoError(t, f.Rmdir(ctx, dirName)) + fstest.CheckListingWithPrecision(t, f, []fstest.Item{}, []string{}, fs.GetModifyWindow(ctx, f)) }) } }) @@ -622,11 +660,11 @@ func Run(t *testing.T, opt *Opt) { t.Run("FsNewObjectNotFound", func(t *testing.T) { skipIfNotOk(t) // Object in an existing directory - o, err := remote.NewObject(ctx, "potato") + o, err := f.NewObject(ctx, "potato") assert.Nil(t, o) assert.Equal(t, fs.ErrorObjectNotFound, err) // Now try an object in a non existing directory - o, err = remote.NewObject(ctx, "directory/not/found/potato") + o, err = f.NewObject(ctx, "directory/not/found/potato") assert.Nil(t, o) assert.Equal(t, fs.ErrorObjectNotFound, err) }) @@ -653,11 +691,11 @@ func Run(t *testing.T, opt *Opt) { in := io.MultiReader(buf, er) obji := object.NewStaticObjectInfo(file2.Path, file2.ModTime, 2*N, true, nil, nil) - _, err := remote.Put(ctx, in, obji) + _, err := f.Put(ctx, in, obji) // assert.Nil(t, obj) - FIXME some remotes return the object even on nil assert.NotNil(t, err) - obj, err := remote.NewObject(ctx, file2.Path) + obj, err := f.NewObject(ctx, file2.Path) assert.Nil(t, obj) assert.Equal(t, fs.ErrorObjectNotFound, err) }) @@ -665,7 +703,7 @@ func Run(t *testing.T, opt *Opt) { t.Run("FsPutZeroLength", func(t *testing.T) { skipIfNotOk(t) - TestPutLarge(ctx, t, remote, &fstest.Item{ + TestPutLarge(ctx, t, f, &fstest.Item{ ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"), Path: fmt.Sprintf("zero-length-file"), Size: int64(0), @@ -674,7 +712,7 @@ func Run(t *testing.T, opt *Opt) { t.Run("FsOpenWriterAt", func(t *testing.T) { skipIfNotOk(t) - openWriterAt := remote.Features().OpenWriterAt + openWriterAt := f.Features().OpenWriterAt if openWriterAt == nil { t.Skip("FS has no OpenWriterAt interface") } @@ -695,11 +733,11 @@ func Run(t *testing.T, opt *Opt) { assert.NoError(t, out.Close()) - obj := findObject(ctx, t, remote, path) + obj := findObject(ctx, t, f, path) assert.Equal(t, "abcdefghi", readObject(ctx, t, obj, -1), "contents of file differ") assert.NoError(t, obj.Remove(ctx)) - assert.NoError(t, remote.Rmdir(ctx, "writer-at-subdir")) + assert.NoError(t, f.Rmdir(ctx, "writer-at-subdir")) }) // TestFsChangeNotify tests that changes are properly @@ -710,12 +748,12 @@ func Run(t *testing.T, opt *Opt) { skipIfNotOk(t) // Check have ChangeNotify - doChangeNotify := remote.Features().ChangeNotify + doChangeNotify := f.Features().ChangeNotify if doChangeNotify == nil { t.Skip("FS has no ChangeNotify interface") } - err := operations.Mkdir(ctx, remote, "dir") + err := operations.Mkdir(ctx, f, "dir") require.NoError(t, err) pollInterval := make(chan time.Duration) @@ -739,7 +777,7 @@ func Run(t *testing.T, opt *Opt) { var dirs []string for _, idx := range []int{1, 3, 2} { dir := fmt.Sprintf("dir/subdir%d", idx) - err = operations.Mkdir(ctx, remote, dir) + err = operations.Mkdir(ctx, f, dir) require.NoError(t, err) dirs = append(dirs, dir) } @@ -750,7 +788,7 @@ func Run(t *testing.T, opt *Opt) { ModTime: time.Now(), Path: fmt.Sprintf("dir/file%d", idx), } - _, o := testPut(ctx, t, remote, &file) + _, o := testPut(ctx, t, f, &file) objs = append(objs, o) } @@ -788,7 +826,7 @@ func Run(t *testing.T, opt *Opt) { } dirs = append(dirs, "dir") for _, dir := range dirs { - assert.NoError(t, remote.Rmdir(ctx, dir)) + assert.NoError(t, f.Rmdir(ctx, dir)) } }) @@ -797,9 +835,9 @@ func Run(t *testing.T, opt *Opt) { // Tests that require file1, file2 are within this t.Run("FsPutFiles", func(t *testing.T) { skipIfNotOk(t) - file1Contents, _ = testPut(ctx, t, remote, &file1) - /* file2Contents = */ testPut(ctx, t, remote, &file2) - file1Contents, _ = testPut(ctx, t, remote, &file1) + file1Contents, _ = testPut(ctx, t, f, &file1) + /* file2Contents = */ testPut(ctx, t, f, &file2) + file1Contents, _ = testPutMimeType(ctx, t, f, &file1, file1MimeType) // Note that the next test will check there are no duplicated file names // TestFsListDirFile2 tests the files are correctly uploaded by doing @@ -809,9 +847,9 @@ func Run(t *testing.T, opt *Opt) { list := func(dir string, expectedDirNames, expectedObjNames []string) { var objNames, dirNames []string for i := 1; i <= *fstest.ListRetries; i++ { - objs, dirs, err := walk.GetAll(ctx, remote, dir, true, 1) + objs, dirs, err := walk.GetAll(ctx, f, dir, true, 1) if errors.Cause(err) == fs.ErrorDirNotFound { - objs, dirs, err = walk.GetAll(ctx, remote, dir, true, 1) + objs, dirs, err = walk.GetAll(ctx, f, dir, true, 1) } require.NoError(t, err) objNames = objsToNames(objs) @@ -857,7 +895,7 @@ func Run(t *testing.T, opt *Opt) { // Test the files are all there with walk.ListR recursive listings t.Run("FsListR", func(t *testing.T) { skipIfNotOk(t) - objs, dirs, err := walk.GetAll(ctx, remote, "", true, -1) + objs, dirs, err := walk.GetAll(ctx, f, "", true, -1) require.NoError(t, err) assert.Equal(t, []string{ "hello? sausage", @@ -875,7 +913,7 @@ func Run(t *testing.T, opt *Opt) { // walk.ListR recursive listings on a sub dir t.Run("FsListRSubdir", func(t *testing.T) { skipIfNotOk(t) - objs, dirs, err := walk.GetAll(ctx, remote, path.Dir(path.Dir(path.Dir(path.Dir(file2.Path)))), true, -1) + objs, dirs, err := walk.GetAll(ctx, f, path.Dir(path.Dir(path.Dir(path.Dir(file2.Path)))), true, -1) require.NoError(t, err) assert.Equal(t, []string{ "hello? sausage/êé", @@ -914,7 +952,7 @@ func Run(t *testing.T, opt *Opt) { for i := 0; i < 2; i++ { dir, _ := path.Split(fileName) dir = dir[:len(dir)-1] - objs, dirs, err = walk.GetAll(ctx, remote, dir, true, -1) + objs, dirs, err = walk.GetAll(ctx, f, dir, true, -1) } require.NoError(t, err) require.Len(t, objs, 1) @@ -932,7 +970,7 @@ func Run(t *testing.T, opt *Opt) { // TestFsListLevel2 tests List works for 2 levels TestFsListLevel2 := func(t *testing.T) { skipIfNotOk(t) - objs, dirs, err := walk.GetAll(ctx, remote, "", true, 2) + objs, dirs, err := walk.GetAll(ctx, f, "", true, 2) if err == fs.ErrorLevelNotSupported { return } @@ -951,27 +989,27 @@ func Run(t *testing.T, opt *Opt) { // TestFsListFile1 tests file present t.Run("FsListFile1", func(t *testing.T) { skipIfNotOk(t) - fstest.CheckListing(t, remote, []fstest.Item{file1, file2}) + fstest.CheckListing(t, f, []fstest.Item{file1, file2}) }) // TestFsNewObject tests NewObject t.Run("FsNewObject", func(t *testing.T) { skipIfNotOk(t) - obj := findObject(ctx, t, remote, file1.Path) - file1.Check(t, obj, remote.Precision()) + obj := findObject(ctx, t, f, file1.Path) + file1.Check(t, obj, f.Precision()) }) // TestFsListFile1and2 tests two files present t.Run("FsListFile1and2", func(t *testing.T) { skipIfNotOk(t) - fstest.CheckListing(t, remote, []fstest.Item{file1, file2}) + fstest.CheckListing(t, f, []fstest.Item{file1, file2}) }) // TestFsNewObjectDir tests NewObject on a directory which should produce an error t.Run("FsNewObjectDir", func(t *testing.T) { skipIfNotOk(t) dir := path.Dir(file2.Path) - obj, err := remote.NewObject(ctx, dir) + obj, err := f.NewObject(ctx, dir) assert.Nil(t, obj) assert.NotNil(t, err) }) @@ -981,7 +1019,7 @@ func Run(t *testing.T, opt *Opt) { skipIfNotOk(t) // Check have Purge - doPurge := remote.Features().Purge + doPurge := f.Features().Purge if doPurge == nil { t.Skip("FS has no Purge interface") } @@ -991,26 +1029,26 @@ func Run(t *testing.T, opt *Opt) { ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"), Path: "dirToPurge/fileToPurge.txt", } - _, _ = testPut(ctx, t, remote, &fileToPurge) + _, _ = testPut(ctx, t, f, &fileToPurge) - fstest.CheckListingWithPrecision(t, remote, []fstest.Item{file1, file2, fileToPurge}, []string{ + fstest.CheckListingWithPrecision(t, f, []fstest.Item{file1, file2, fileToPurge}, []string{ "dirToPurge", "hello? sausage", "hello? sausage/êé", "hello? sausage/êé/Hello, 世界", "hello? sausage/êé/Hello, 世界/ \" ' @ < > & ? + ≠", - }, fs.GetModifyWindow(ctx, remote)) + }, fs.GetModifyWindow(ctx, f)) // Now purge it - err = operations.Purge(ctx, remote, "dirToPurge") + err = operations.Purge(ctx, f, "dirToPurge") require.NoError(t, err) - fstest.CheckListingWithPrecision(t, remote, []fstest.Item{file1, file2}, []string{ + fstest.CheckListingWithPrecision(t, f, []fstest.Item{file1, file2}, []string{ "hello? sausage", "hello? sausage/êé", "hello? sausage/êé/Hello, 世界", "hello? sausage/êé/Hello, 世界/ \" ' @ < > & ? + ≠", - }, fs.GetModifyWindow(ctx, remote)) + }, fs.GetModifyWindow(ctx, f)) }) // TestFsCopy tests Copy @@ -1018,7 +1056,7 @@ func Run(t *testing.T, opt *Opt) { skipIfNotOk(t) // Check have Copy - doCopy := remote.Features().Copy + doCopy := f.Features().Copy if doCopy == nil { t.Skip("FS has no Copier interface") } @@ -1028,7 +1066,7 @@ func Run(t *testing.T, opt *Opt) { file2Copy.Path += "-copy" // do the copy - src := findObject(ctx, t, remote, file2.Path) + src := findObject(ctx, t, f, file2.Path) dst, err := doCopy(ctx, src, file2Copy.Path) if err == fs.ErrorCantCopy { t.Skip("FS can't copy") @@ -1036,7 +1074,7 @@ func Run(t *testing.T, opt *Opt) { require.NoError(t, err, fmt.Sprintf("Error: %#v", err)) // check file exists in new listing - fstest.CheckListing(t, remote, []fstest.Item{file1, file2, file2Copy}) + fstest.CheckListing(t, f, []fstest.Item{file1, file2, file2Copy}) // Check dst lightly - list above has checked ModTime/Hashes assert.Equal(t, file2Copy.Path, dst.Remote()) @@ -1052,7 +1090,7 @@ func Run(t *testing.T, opt *Opt) { skipIfNotOk(t) // Check have Move - doMove := remote.Features().Move + doMove := f.Features().Move if doMove == nil { t.Skip("FS has no Mover interface") } @@ -1067,14 +1105,14 @@ func Run(t *testing.T, opt *Opt) { // check happy path, i.e. no naming conflicts when rename and move are two // separate operations file2Move.Path = "other.txt" - src := findObject(ctx, t, remote, file2.Path) + src := findObject(ctx, t, f, file2.Path) dst, err := doMove(ctx, src, file2Move.Path) if err == fs.ErrorCantMove { t.Skip("FS can't move") } require.NoError(t, err) // check file exists in new listing - fstest.CheckListing(t, remote, []fstest.Item{file1, file2Move}) + fstest.CheckListing(t, f, []fstest.Item{file1, file2Move}) // Check dst lightly - list above has checked ModTime/Hashes assert.Equal(t, file2Move.Path, dst.Remote()) // 1: file name.txt @@ -1082,30 +1120,30 @@ func Run(t *testing.T, opt *Opt) { // Check conflict on "rename, then move" file1Move.Path = "moveTest/other.txt" - src = findObject(ctx, t, remote, file1.Path) + src = findObject(ctx, t, f, file1.Path) _, err = doMove(ctx, src, file1Move.Path) require.NoError(t, err) - fstest.CheckListing(t, remote, []fstest.Item{file1Move, file2Move}) + fstest.CheckListing(t, f, []fstest.Item{file1Move, file2Move}) // 1: moveTest/other.txt // 2: other.txt // Check conflict on "move, then rename" - src = findObject(ctx, t, remote, file1Move.Path) + src = findObject(ctx, t, f, file1Move.Path) _, err = doMove(ctx, src, file1.Path) require.NoError(t, err) - fstest.CheckListing(t, remote, []fstest.Item{file1, file2Move}) + fstest.CheckListing(t, f, []fstest.Item{file1, file2Move}) // 1: file name.txt // 2: other.txt - src = findObject(ctx, t, remote, file2Move.Path) + src = findObject(ctx, t, f, file2Move.Path) _, err = doMove(ctx, src, file2.Path) require.NoError(t, err) - fstest.CheckListing(t, remote, []fstest.Item{file1, file2}) + fstest.CheckListing(t, f, []fstest.Item{file1, file2}) // 1: file name.txt // 2: hello sausage?/../z.txt // Tidy up moveTest directory - require.NoError(t, remote.Rmdir(ctx, "moveTest")) + require.NoError(t, f.Rmdir(ctx, "moveTest")) }) // Move src to this remote using server-side move operations. @@ -1123,13 +1161,13 @@ func Run(t *testing.T, opt *Opt) { skipIfNotOk(t) // Check have DirMove - doDirMove := remote.Features().DirMove + doDirMove := f.Features().DirMove if doDirMove == nil { t.Skip("FS has no DirMover interface") } // Check it can't move onto itself - err := doDirMove(ctx, remote, "", "") + err := doDirMove(ctx, f, "", "") require.Equal(t, fs.ErrorDirExists, err) // new remote @@ -1139,12 +1177,12 @@ func Run(t *testing.T, opt *Opt) { const newName = "new_name/sub_new_name" // try the move - err = newRemote.Features().DirMove(ctx, remote, "", newName) + err = newRemote.Features().DirMove(ctx, f, "", newName) require.NoError(t, err) // check remotes // remote should not exist here - _, err = remote.List(ctx, "") + _, err = f.List(ctx, "") assert.Equal(t, fs.ErrorDirNotFound, errors.Cause(err)) //fstest.CheckListingWithPrecision(t, remote, []fstest.Item{}, []string{}, remote.Precision()) file1Copy := file1 @@ -1165,12 +1203,12 @@ func Run(t *testing.T, opt *Opt) { require.NoError(t, err) // check remotes - fstest.CheckListingWithPrecision(t, remote, []fstest.Item{file2, file1}, []string{ + fstest.CheckListingWithPrecision(t, f, []fstest.Item{file2, file1}, []string{ "hello? sausage", "hello? sausage/êé", "hello? sausage/êé/Hello, 世界", "hello? sausage/êé/Hello, 世界/ \" ' @ < > & ? + ≠", - }, remote.Precision()) + }, f.Precision()) fstest.CheckListingWithPrecision(t, newRemote, []fstest.Item{}, []string{ "new_name", }, newRemote.Precision()) @@ -1179,17 +1217,17 @@ func Run(t *testing.T, opt *Opt) { // TestFsRmdirFull tests removing a non empty directory t.Run("FsRmdirFull", func(t *testing.T) { skipIfNotOk(t) - if isBucketBasedButNotRoot(remote) { + if isBucketBasedButNotRoot(f) { t.Skip("Skipping test as non root bucket based remote") } - err := remote.Rmdir(ctx, "") + err := f.Rmdir(ctx, "") require.Error(t, err, "Expecting error on RMdir on non empty remote") }) // TestFsPrecision tests the Precision of the Fs t.Run("FsPrecision", func(t *testing.T) { skipIfNotOk(t) - precision := remote.Precision() + precision := f.Precision() if precision == fs.ModTimeNotSupported { return } @@ -1202,7 +1240,7 @@ func Run(t *testing.T, opt *Opt) { // TestObjectString tests the Object String method t.Run("ObjectString", func(t *testing.T) { skipIfNotOk(t) - obj := findObject(ctx, t, remote, file1.Path) + obj := findObject(ctx, t, f, file1.Path) assert.Equal(t, file1.Path, obj.String()) if opt.NilObject != nil { assert.Equal(t, "", opt.NilObject.String()) @@ -1212,13 +1250,13 @@ func Run(t *testing.T, opt *Opt) { // TestObjectFs tests the object can be found t.Run("ObjectFs", func(t *testing.T) { skipIfNotOk(t) - obj := findObject(ctx, t, remote, file1.Path) + obj := findObject(ctx, t, f, file1.Path) // If this is set we don't do the direct comparison of // the Fs from the object as it may be different if opt.SkipFsMatch { return } - testRemote := remote + testRemote := f if obj.Fs() != testRemote { // Check to see if this wraps something else if doUnWrap := testRemote.Features().UnWrap; doUnWrap != nil { @@ -1231,38 +1269,44 @@ func Run(t *testing.T, opt *Opt) { // TestObjectRemote tests the Remote is correct t.Run("ObjectRemote", func(t *testing.T) { skipIfNotOk(t) - obj := findObject(ctx, t, remote, file1.Path) + obj := findObject(ctx, t, f, file1.Path) assert.Equal(t, file1.Path, obj.Remote()) }) // TestObjectHashes checks all the hashes the object supports t.Run("ObjectHashes", func(t *testing.T) { skipIfNotOk(t) - obj := findObject(ctx, t, remote, file1.Path) + obj := findObject(ctx, t, f, file1.Path) file1.CheckHashes(t, obj) }) // TestObjectModTime tests the ModTime of the object is correct TestObjectModTime := func(t *testing.T) { skipIfNotOk(t) - obj := findObject(ctx, t, remote, file1.Path) - file1.CheckModTime(t, obj, obj.ModTime(ctx), remote.Precision()) + obj := findObject(ctx, t, f, file1.Path) + file1.CheckModTime(t, obj, obj.ModTime(ctx), f.Precision()) } t.Run("ObjectModTime", TestObjectModTime) // TestObjectMimeType tests the MimeType of the object is correct t.Run("ObjectMimeType", func(t *testing.T) { skipIfNotOk(t) - obj := findObject(ctx, t, remote, file1.Path) + features := f.Features() + obj := findObject(ctx, t, f, file1.Path) do, ok := obj.(fs.MimeTyper) + require.Equal(t, features.ReadMimeType, ok, "mismatch between Object.MimeType and Features.ReadMimeType") if !ok { t.Skip("MimeType method not supported") } mimeType := do.MimeType(ctx) - if strings.ContainsRune(mimeType, ';') { - assert.Equal(t, "text/plain; charset=utf-8", mimeType) + if features.WriteMimeType { + assert.Equal(t, file1MimeType, mimeType, "can read and write mime types but failed") } else { - assert.Equal(t, "text/plain", mimeType) + if strings.ContainsRune(mimeType, ';') { + assert.Equal(t, "text/plain; charset=utf-8", mimeType) + } else { + assert.Equal(t, "text/plain", mimeType) + } } }) @@ -1270,7 +1314,7 @@ func Run(t *testing.T, opt *Opt) { t.Run("ObjectSetModTime", func(t *testing.T) { skipIfNotOk(t) newModTime := fstest.Time("2011-12-13T14:15:16.999999999Z") - obj := findObject(ctx, t, remote, file1.Path) + obj := findObject(ctx, t, f, file1.Path) err := obj.SetModTime(ctx, newModTime) if err == fs.ErrorCantSetModTime || err == fs.ErrorCantSetModTimeWithoutDelete { t.Log(err) @@ -1278,7 +1322,7 @@ func Run(t *testing.T, opt *Opt) { } require.NoError(t, err) file1.ModTime = newModTime - file1.CheckModTime(t, obj, obj.ModTime(ctx), remote.Precision()) + file1.CheckModTime(t, obj, obj.ModTime(ctx), f.Precision()) // And make a new object and read it from there too TestObjectModTime(t) }) @@ -1286,21 +1330,21 @@ func Run(t *testing.T, opt *Opt) { // TestObjectSize tests that Size works t.Run("ObjectSize", func(t *testing.T) { skipIfNotOk(t) - obj := findObject(ctx, t, remote, file1.Path) + obj := findObject(ctx, t, f, file1.Path) assert.Equal(t, file1.Size, obj.Size()) }) // TestObjectOpen tests that Open works t.Run("ObjectOpen", func(t *testing.T) { skipIfNotOk(t) - obj := findObject(ctx, t, remote, file1.Path) + obj := findObject(ctx, t, f, file1.Path) assert.Equal(t, file1Contents, readObject(ctx, t, obj, -1), "contents of file1 differ") }) // TestObjectOpenSeek tests that Open works with SeekOption t.Run("ObjectOpenSeek", func(t *testing.T) { skipIfNotOk(t) - obj := findObject(ctx, t, remote, file1.Path) + obj := findObject(ctx, t, f, file1.Path) assert.Equal(t, file1Contents[50:], readObject(ctx, t, obj, -1, &fs.SeekOption{Offset: 50}), "contents of file1 differ after seek") }) @@ -1309,7 +1353,7 @@ func Run(t *testing.T, opt *Opt) { // go test -v -run 'TestIntegration/Test(Setup|Init|FsMkdir|FsPutFile1|FsPutFile2|FsUpdateFile1|ObjectOpenRange)$' t.Run("ObjectOpenRange", func(t *testing.T) { skipIfNotOk(t) - obj := findObject(ctx, t, remote, file1.Path) + obj := findObject(ctx, t, f, file1.Path) for _, test := range []struct { ro fs.RangeOption wantStart, wantEnd int @@ -1330,7 +1374,7 @@ func Run(t *testing.T, opt *Opt) { // TestObjectPartialRead tests that reading only part of the object does the correct thing t.Run("ObjectPartialRead", func(t *testing.T) { skipIfNotOk(t) - obj := findObject(ctx, t, remote, file1.Path) + obj := findObject(ctx, t, f, file1.Path) assert.Equal(t, file1Contents[:50], readObject(ctx, t, obj, 50), "contents of file1 differ after limited read") }) @@ -1343,18 +1387,18 @@ func Run(t *testing.T, opt *Opt) { in := io.TeeReader(buf, hash) file1.Size = int64(buf.Len()) - obj := findObject(ctx, t, remote, file1.Path) + obj := findObject(ctx, t, f, file1.Path) obji := object.NewStaticObjectInfo(file1.Path, file1.ModTime, int64(len(contents)), true, nil, obj.Fs()) err := obj.Update(ctx, in, obji) require.NoError(t, err) file1.Hashes = hash.Sums() // check the object has been updated - file1.Check(t, obj, remote.Precision()) + file1.Check(t, obj, f.Precision()) // Re-read the object and check again - obj = findObject(ctx, t, remote, file1.Path) - file1.Check(t, obj, remote.Precision()) + obj = findObject(ctx, t, f, file1.Path) + file1.Check(t, obj, f.Precision()) // check contents correct assert.Equal(t, contents, readObject(ctx, t, obj, -1), "contents of updated file1 differ") @@ -1364,7 +1408,7 @@ func Run(t *testing.T, opt *Opt) { // TestObjectStorable tests that Storable works t.Run("ObjectStorable", func(t *testing.T) { skipIfNotOk(t) - obj := findObject(ctx, t, remote, file1.Path) + obj := findObject(ctx, t, f, file1.Path) require.NotNil(t, !obj.Storable(), "Expecting object to be storable") }) @@ -1398,7 +1442,7 @@ func Run(t *testing.T, opt *Opt) { // Test that things work from the root t.Run("FromRoot", func(t *testing.T) { - if features := remote.Features(); features.BucketBased && !features.BucketBasedRootOK { + if features := f.Features(); features.BucketBased && !features.BucketBasedRootOK { t.Skip("Can't list from root on this remote") } @@ -1500,7 +1544,7 @@ func Run(t *testing.T, opt *Opt) { t.Run("PublicLink", func(t *testing.T) { skipIfNotOk(t) - doPublicLink := remote.Features().PublicLink + doPublicLink := f.Features().PublicLink if doPublicLink == nil { t.Skip("FS has no PublicLinker interface") } @@ -1561,7 +1605,7 @@ func Run(t *testing.T, opt *Opt) { // TestSetTier tests SetTier and GetTier functionality t.Run("SetTier", func(t *testing.T) { skipIfNotSetTier(t) - obj := findObject(ctx, t, remote, file1.Path) + obj := findObject(ctx, t, f, file1.Path) setter, ok := obj.(fs.SetTierer) assert.NotNil(t, ok) getter, ok := obj.(fs.GetTierer) @@ -1585,11 +1629,11 @@ func Run(t *testing.T, opt *Opt) { if opt.SkipObjectCheckWrap { t.Skip("Skipping FsCheckWrap on this Fs") } - ft := new(fs.Features).Fill(ctx, remote) + ft := new(fs.Features).Fill(ctx, f) if ft.UnWrap == nil { t.Skip("Not a wrapping Fs") } - obj := findObject(ctx, t, remote, file1.Path) + obj := findObject(ctx, t, f, file1.Path) _, unsupported := fs.ObjectOptionalInterfaces(obj) for _, name := range unsupported { if !stringsContains(name, opt.UnimplementableObjectMethods) { @@ -1602,11 +1646,11 @@ func Run(t *testing.T, opt *Opt) { t.Run("ObjectRemove", func(t *testing.T) { skipIfNotOk(t) // remove file1 - obj := findObject(ctx, t, remote, file1.Path) + obj := findObject(ctx, t, f, file1.Path) err := obj.Remove(ctx) require.NoError(t, err) // check listing without modtime as TestPublicLink may change the modtime - fstest.CheckListingWithPrecision(t, remote, []fstest.Item{file2}, nil, fs.ModTimeNotSupported) + fstest.CheckListingWithPrecision(t, f, []fstest.Item{file2}, nil, fs.ModTimeNotSupported) }) // TestAbout tests the About optional interface @@ -1614,7 +1658,7 @@ func Run(t *testing.T, opt *Opt) { skipIfNotOk(t) // Check have About - doAbout := remote.Features().About + doAbout := f.Features().About if doAbout == nil { t.Skip("FS does not support About") } @@ -1633,7 +1677,7 @@ func Run(t *testing.T, opt *Opt) { // close to the end of suite. (See fs/operations/xtra_operations_test.go) t.Run("FsPutStream", func(t *testing.T) { skipIfNotOk(t) - if remote.Features().PutStream == nil { + if f.Features().PutStream == nil { t.Skip("FS has no PutStream interface") } @@ -1658,15 +1702,15 @@ func Run(t *testing.T, opt *Opt) { file.Size = -1 obji := object.NewStaticObjectInfo(file.Path, file.ModTime, file.Size, true, nil, nil) - obj, err = remote.Features().PutStream(ctx, in, obji) + obj, err = f.Features().PutStream(ctx, in, obji) return err }) file.Hashes = uploadHash.Sums() file.Size = int64(contentSize) // use correct size when checking - file.Check(t, obj, remote.Precision()) + file.Check(t, obj, f.Precision()) // Re-read the object and check again - obj = findObject(ctx, t, remote, file.Path) - file.Check(t, obj, remote.Precision()) + obj = findObject(ctx, t, f, file.Path) + file.Check(t, obj, f.Precision()) require.NoError(t, obj.Remove(ctx)) }) } @@ -1675,10 +1719,10 @@ func Run(t *testing.T, opt *Opt) { // TestInternal calls InternalTest() on the Fs t.Run("Internal", func(t *testing.T) { skipIfNotOk(t) - if it, ok := remote.(InternalTester); ok { + if it, ok := f.(InternalTester); ok { it.InternalTest(t) } else { - t.Skipf("%T does not implement InternalTester", remote) + t.Skipf("%T does not implement InternalTester", f) } }) @@ -1693,12 +1737,12 @@ func Run(t *testing.T, opt *Opt) { t.Skip("not running with -short") } - setUploadChunkSizer, _ := remote.(SetUploadChunkSizer) + setUploadChunkSizer, _ := f.(SetUploadChunkSizer) if setUploadChunkSizer == nil { - t.Skipf("%T does not implement SetUploadChunkSizer", remote) + t.Skipf("%T does not implement SetUploadChunkSizer", f) } - setUploadCutoffer, _ := remote.(SetUploadCutoffer) + setUploadCutoffer, _ := f.(SetUploadCutoffer) minChunkSize := opt.ChunkedUpload.MinChunkSize if minChunkSize < 100 { @@ -1782,7 +1826,7 @@ func Run(t *testing.T, opt *Opt) { for _, fileSize := range testChunks { t.Run(fmt.Sprintf("%d", fileSize), func(t *testing.T) { - TestPutLarge(ctx, t, remote, &fstest.Item{ + TestPutLarge(ctx, t, f, &fstest.Item{ ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"), Path: fmt.Sprintf("chunked-%s-%s.bin", cs.String(), fileSize.String()), Size: int64(fileSize), @@ -1810,7 +1854,7 @@ func Run(t *testing.T, opt *Opt) { in := bytes.NewBufferString(contents) obji := object.NewStaticObjectInfo("unknown-size-put.txt", fstest.Time("2002-02-03T04:05:06.499999999Z"), -1, true, nil, nil) - obj, err := remote.Put(ctx, in, obji) + obj, err := f.Put(ctx, in, obji) if err == nil { require.NoError(t, obj.Remove(ctx), "successfully uploaded unknown-sized file but failed to remove") } @@ -1823,7 +1867,7 @@ func Run(t *testing.T, opt *Opt) { Path: "unknown-size-update.txt", } - testPut(ctx, t, remote, &unknownSizeUpdateFile) + testPut(ctx, t, f, &unknownSizeUpdateFile) defer func() { assert.Nil(t, recover(), "Object.Update() should not panic when src.Size() == -1") @@ -1832,7 +1876,7 @@ func Run(t *testing.T, opt *Opt) { newContents := random.String(200) in := bytes.NewBufferString(newContents) - obj := findObject(ctx, t, remote, unknownSizeUpdateFile.Path) + obj := findObject(ctx, t, f, unknownSizeUpdateFile.Path) obji := object.NewStaticObjectInfo(unknownSizeUpdateFile.Path, unknownSizeUpdateFile.ModTime, -1, true, nil, obj.Fs()) err := obj.Update(ctx, in, obji) if err == nil { @@ -1865,16 +1909,16 @@ func Run(t *testing.T, opt *Opt) { }) // Purge the folder - err = operations.Purge(ctx, remote, "") + err = operations.Purge(ctx, f, "") if errors.Cause(err) != fs.ErrorDirNotFound { require.NoError(t, err) } purged = true - fstest.CheckListing(t, remote, []fstest.Item{}) + fstest.CheckListing(t, f, []fstest.Item{}) // Check purging again if not bucket based - if !isBucketBasedButNotRoot(remote) { - err = operations.Purge(ctx, remote, "") + if !isBucketBasedButNotRoot(f) { + err = operations.Purge(ctx, f, "") assert.Error(t, err, "Expecting error after on second purge") if errors.Cause(err) != fs.ErrorDirNotFound { t.Log("Warning: this should produce fs.ErrorDirNotFound") @@ -1885,7 +1929,7 @@ func Run(t *testing.T, opt *Opt) { // Check directory is purged if !purged { - _ = operations.Purge(ctx, remote, "") + _ = operations.Purge(ctx, f, "") } // Remove the local directory so we don't clutter up /tmp