azureblob,memory,pcloud: fix setting of mime types

Before this change the backend was reading the mime type of the
destination object instead of the source object when uploading.

This changes fixes the problem and introduces an integration test for
it.

See: https://forum.rclone.org/t/is-there-a-way-to-get-rclone-copy-to-preserve-metadata/20682/2
This commit is contained in:
Nick Craig-Wood 2020-11-26 21:28:39 +00:00
parent 19a8b66cee
commit dfadd98969
4 changed files with 195 additions and 151 deletions

View File

@ -1453,7 +1453,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
blob := o.getBlobReference() blob := o.getBlobReference()
httpHeaders := azblob.BlobHTTPHeaders{} httpHeaders := azblob.BlobHTTPHeaders{}
httpHeaders.ContentType = fs.MimeType(ctx, o) httpHeaders.ContentType = fs.MimeType(ctx, src)
// Compute the Content-MD5 of the file, for multiparts uploads it // Compute the Content-MD5 of the file, for multiparts uploads it
// will be set in PutBlockList API call using the 'x-ms-blob-content-md5' header // will be set in PutBlockList API call using the 'x-ms-blob-content-md5' header
// Note: If multipart, an MD5 checksum will also be computed for each uploaded block // Note: If multipart, an MD5 checksum will also be computed for each uploaded block

View File

@ -592,7 +592,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
data: data, data: data,
hash: "", hash: "",
modTime: src.ModTime(ctx), modTime: src.ModTime(ctx),
mimeType: fs.MimeType(ctx, o), mimeType: fs.MimeType(ctx, src),
} }
buckets.updateObjectData(bucket, bucketPath, o.od) buckets.updateObjectData(bucket, bucketPath, o.od)
return nil return nil

View File

@ -1117,7 +1117,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
Method: "PUT", Method: "PUT",
Path: "/uploadfile", Path: "/uploadfile",
Body: in, Body: in,
ContentType: fs.MimeType(ctx, o), ContentType: fs.MimeType(ctx, src),
ContentLength: &size, ContentLength: &size,
Parameters: url.Values{}, Parameters: url.Values{},
TransferEncoding: []string{"identity"}, // pcloud doesn't like chunked encoding TransferEncoding: []string{"identity"}, // pcloud doesn't like chunked encoding

View File

@ -153,13 +153,32 @@ func retry(t *testing.T, what string, f func() error) {
require.NoError(t, err, what) require.NoError(t, err, what)
} }
// testPut puts file with random contents to the remote // An fs.ObjectInfo that can override mime type
func testPut(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item) (string, fs.Object) { type objectInfoWithMimeType struct {
return PutTestContents(ctx, t, f, file, random.String(100), true) fs.ObjectInfo
mimeType string
} }
// PutTestContents puts file with given contents to the remote and checks it but unlike TestPutLarge doesn't remove // Return a wrapped fs.ObjectInfo which returns the mime type given
func PutTestContents(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item, contents string, check bool) (string, fs.Object) { func overrideMimeType(o fs.ObjectInfo, mimeType string) fs.ObjectInfo {
return &objectInfoWithMimeType{
ObjectInfo: o,
mimeType: mimeType,
}
}
// MimeType that was overridden
func (o *objectInfoWithMimeType) MimeType(ctx context.Context) string {
return o.mimeType
}
// check interface
var _ fs.MimeTyper = (*objectInfoWithMimeType)(nil)
// putTestContentsMimeType puts file with given contents to the remote and checks it but unlike TestPutLarge doesn't remove
//
// it uploads the object with the mimeType passed in if set
func putTestContentsMimeType(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item, contents string, check bool, mimeType string) (string, fs.Object) {
var ( var (
err error err error
obj fs.Object obj fs.Object
@ -172,6 +191,9 @@ func PutTestContents(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.It
file.Size = int64(buf.Len()) file.Size = int64(buf.Len())
obji := object.NewStaticObjectInfo(file.Path, file.ModTime, file.Size, true, nil, nil) obji := object.NewStaticObjectInfo(file.Path, file.ModTime, file.Size, true, nil, nil)
if mimeType != "" {
obji = overrideMimeType(obji, mimeType)
}
obj, err = f.Put(ctx, in, obji) obj, err = f.Put(ctx, in, obji)
return err return err
}) })
@ -185,6 +207,21 @@ func PutTestContents(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.It
return contents, obj return contents, obj
} }
// PutTestContents puts file with given contents to the remote and checks it but unlike TestPutLarge doesn't remove
func PutTestContents(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item, contents string, check bool) (string, fs.Object) {
return putTestContentsMimeType(ctx, t, f, file, contents, check, "")
}
// testPut puts file with random contents to the remote
func testPut(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item) (string, fs.Object) {
return PutTestContents(ctx, t, f, file, random.String(100), true)
}
// testPutMimeType puts file with random contents to the remote and the mime type given
func testPutMimeType(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item, mimeType string) (string, fs.Object) {
return putTestContentsMimeType(ctx, t, f, file, random.String(100), true, mimeType)
}
// TestPutLarge puts file to the remote, checks it and removes it on success. // TestPutLarge puts file to the remote, checks it and removes it on success.
func TestPutLarge(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item) { func TestPutLarge(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item) {
var ( var (
@ -279,7 +316,7 @@ func stringsContains(x string, ss []string) bool {
// - these are inside the "FsPutFiles" test. // - these are inside the "FsPutFiles" test.
func Run(t *testing.T, opt *Opt) { func Run(t *testing.T, opt *Opt) {
var ( var (
remote fs.Fs f fs.Fs
remoteName = opt.RemoteName remoteName = opt.RemoteName
subRemoteName string subRemoteName string
subRemoteLeaf string subRemoteLeaf string
@ -288,6 +325,7 @@ func Run(t *testing.T, opt *Opt) {
Path: "file name.txt", Path: "file name.txt",
} }
file1Contents string file1Contents string
file1MimeType = "text/csv"
file2 = fstest.Item{ file2 = fstest.Item{
ModTime: fstest.Time("2001-02-03T04:05:10.123123123Z"), ModTime: fstest.Time("2001-02-03T04:05:10.123123123Z"),
Path: `hello? sausage/êé/Hello, 世界/ " ' @ < > & ? + ≠/z.txt`, Path: `hello? sausage/êé/Hello, 世界/ " ' @ < > & ? + ≠/z.txt`,
@ -305,7 +343,7 @@ func Run(t *testing.T, opt *Opt) {
// Skip the test if the remote isn't configured // Skip the test if the remote isn't configured
skipIfNotOk := func(t *testing.T) { skipIfNotOk := func(t *testing.T) {
if remote == nil { if f == nil {
t.Skipf("WARN: %q not configured", remoteName) t.Skipf("WARN: %q not configured", remoteName)
} }
} }
@ -314,7 +352,7 @@ func Run(t *testing.T, opt *Opt) {
// flag, returning a function to restore its value // flag, returning a function to restore its value
skipIfNotListR := func(t *testing.T) func() { skipIfNotListR := func(t *testing.T) func() {
skipIfNotOk(t) skipIfNotOk(t)
if remote.Features().ListR == nil { if f.Features().ListR == nil {
t.Skip("FS has no ListR interface") t.Skip("FS has no ListR interface")
} }
previous := ci.UseListR previous := ci.UseListR
@ -327,8 +365,8 @@ func Run(t *testing.T, opt *Opt) {
// Skip if remote is not SetTier and GetTier capable // Skip if remote is not SetTier and GetTier capable
skipIfNotSetTier := func(t *testing.T) { skipIfNotSetTier := func(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
if remote.Features().SetTier == false || if f.Features().SetTier == false ||
remote.Features().GetTier == false { f.Features().GetTier == false {
t.Skip("FS has no SetTier & GetTier interfaces") t.Skip("FS has no SetTier & GetTier interfaces")
} }
} }
@ -374,7 +412,7 @@ func Run(t *testing.T, opt *Opt) {
// remote - the result of fs.NewFs(TestRemote:subRemoteName) // remote - the result of fs.NewFs(TestRemote:subRemoteName)
subRemoteName, subRemoteLeaf, err = fstest.RandomRemoteName(remoteName) subRemoteName, subRemoteLeaf, err = fstest.RandomRemoteName(remoteName)
require.NoError(t, err) require.NoError(t, err)
remote, err = fs.NewFs(context.Background(), subRemoteName) f, err = fs.NewFs(context.Background(), subRemoteName)
if err == fs.ErrorNotFoundInConfigFile { if err == fs.ErrorNotFoundInConfigFile {
t.Logf("Didn't find %q in config file - skipping tests", remoteName) t.Logf("Didn't find %q in config file - skipping tests", remoteName)
return return
@ -390,7 +428,7 @@ func Run(t *testing.T, opt *Opt) {
if opt.SkipFsCheckWrap { if opt.SkipFsCheckWrap {
t.Skip("Skipping FsCheckWrap on this Fs") t.Skip("Skipping FsCheckWrap on this Fs")
} }
ft := new(fs.Features).Fill(ctx, remote) ft := new(fs.Features).Fill(ctx, f)
if ft.UnWrap == nil { if ft.UnWrap == nil {
t.Skip("Not a wrapping Fs") t.Skip("Not a wrapping Fs")
} }
@ -418,7 +456,7 @@ func Run(t *testing.T, opt *Opt) {
// Check to see if Fs advertises commands and they work and have docs // Check to see if Fs advertises commands and they work and have docs
t.Run("FsCommand", func(t *testing.T) { t.Run("FsCommand", func(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
doCommand := remote.Features().Command doCommand := f.Features().Command
if doCommand == nil { if doCommand == nil {
t.Skip("No commands in this remote") t.Skip("No commands in this remote")
} }
@ -434,29 +472,29 @@ func Run(t *testing.T, opt *Opt) {
// TestFsRmdirNotFound tests deleting a non existent directory // TestFsRmdirNotFound tests deleting a non existent directory
t.Run("FsRmdirNotFound", func(t *testing.T) { t.Run("FsRmdirNotFound", func(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
if isBucketBasedButNotRoot(remote) { if isBucketBasedButNotRoot(f) {
t.Skip("Skipping test as non root bucket based remote") t.Skip("Skipping test as non root bucket based remote")
} }
err := remote.Rmdir(ctx, "") err := f.Rmdir(ctx, "")
assert.Error(t, err, "Expecting error on Rmdir non existent") assert.Error(t, err, "Expecting error on Rmdir non existent")
}) })
// Make the directory // Make the directory
err = remote.Mkdir(ctx, "") err = f.Mkdir(ctx, "")
require.NoError(t, err) require.NoError(t, err)
fstest.CheckListing(t, remote, []fstest.Item{}) fstest.CheckListing(t, f, []fstest.Item{})
// TestFsString tests the String method // TestFsString tests the String method
t.Run("FsString", func(t *testing.T) { t.Run("FsString", func(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
str := remote.String() str := f.String()
require.NotEqual(t, "", str) require.NotEqual(t, "", str)
}) })
// TestFsName tests the Name method // TestFsName tests the Name method
t.Run("FsName", func(t *testing.T) { t.Run("FsName", func(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
got := remote.Name() got := f.Name()
want := remoteName[:strings.LastIndex(remoteName, ":")+1] want := remoteName[:strings.LastIndex(remoteName, ":")+1]
if isLocalRemote { if isLocalRemote {
want = "local:" want = "local:"
@ -467,8 +505,8 @@ func Run(t *testing.T, opt *Opt) {
// TestFsRoot tests the Root method // TestFsRoot tests the Root method
t.Run("FsRoot", func(t *testing.T) { t.Run("FsRoot", func(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
name := remote.Name() + ":" name := f.Name() + ":"
root := remote.Root() root := f.Root()
if isLocalRemote { if isLocalRemote {
// only check last path element on local // only check last path element on local
require.Equal(t, filepath.Base(subRemoteName), filepath.Base(root)) require.Equal(t, filepath.Base(subRemoteName), filepath.Base(root))
@ -480,7 +518,7 @@ func Run(t *testing.T, opt *Opt) {
// TestFsRmdirEmpty tests deleting an empty directory // TestFsRmdirEmpty tests deleting an empty directory
t.Run("FsRmdirEmpty", func(t *testing.T) { t.Run("FsRmdirEmpty", func(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
err := remote.Rmdir(ctx, "") err := f.Rmdir(ctx, "")
require.NoError(t, err) require.NoError(t, err)
}) })
@ -490,41 +528,41 @@ func Run(t *testing.T, opt *Opt) {
t.Run("FsMkdir", func(t *testing.T) { t.Run("FsMkdir", func(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
err := remote.Mkdir(ctx, "") err := f.Mkdir(ctx, "")
require.NoError(t, err) require.NoError(t, err)
fstest.CheckListing(t, remote, []fstest.Item{}) fstest.CheckListing(t, f, []fstest.Item{})
err = remote.Mkdir(ctx, "") err = f.Mkdir(ctx, "")
require.NoError(t, err) require.NoError(t, err)
// TestFsMkdirRmdirSubdir tests making and removing a sub directory // TestFsMkdirRmdirSubdir tests making and removing a sub directory
t.Run("FsMkdirRmdirSubdir", func(t *testing.T) { t.Run("FsMkdirRmdirSubdir", func(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
dir := "dir/subdir" dir := "dir/subdir"
err := operations.Mkdir(ctx, remote, dir) err := operations.Mkdir(ctx, f, dir)
require.NoError(t, err) require.NoError(t, err)
fstest.CheckListingWithPrecision(t, remote, []fstest.Item{}, []string{"dir", "dir/subdir"}, fs.GetModifyWindow(ctx, remote)) fstest.CheckListingWithPrecision(t, f, []fstest.Item{}, []string{"dir", "dir/subdir"}, fs.GetModifyWindow(ctx, f))
err = operations.Rmdir(ctx, remote, dir) err = operations.Rmdir(ctx, f, dir)
require.NoError(t, err) require.NoError(t, err)
fstest.CheckListingWithPrecision(t, remote, []fstest.Item{}, []string{"dir"}, fs.GetModifyWindow(ctx, remote)) fstest.CheckListingWithPrecision(t, f, []fstest.Item{}, []string{"dir"}, fs.GetModifyWindow(ctx, f))
err = operations.Rmdir(ctx, remote, "dir") err = operations.Rmdir(ctx, f, "dir")
require.NoError(t, err) require.NoError(t, err)
fstest.CheckListingWithPrecision(t, remote, []fstest.Item{}, []string{}, fs.GetModifyWindow(ctx, remote)) fstest.CheckListingWithPrecision(t, f, []fstest.Item{}, []string{}, fs.GetModifyWindow(ctx, f))
}) })
// TestFsListEmpty tests listing an empty directory // TestFsListEmpty tests listing an empty directory
t.Run("FsListEmpty", func(t *testing.T) { t.Run("FsListEmpty", func(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
fstest.CheckListing(t, remote, []fstest.Item{}) fstest.CheckListing(t, f, []fstest.Item{})
}) })
// TestFsListDirEmpty tests listing the directories from an empty directory // TestFsListDirEmpty tests listing the directories from an empty directory
TestFsListDirEmpty := func(t *testing.T) { TestFsListDirEmpty := func(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
objs, dirs, err := walk.GetAll(ctx, remote, "", true, 1) objs, dirs, err := walk.GetAll(ctx, f, "", true, 1)
if !remote.Features().CanHaveEmptyDirectories { if !f.Features().CanHaveEmptyDirectories {
if err != fs.ErrorDirNotFound { if err != fs.ErrorDirNotFound {
require.NoError(t, err) require.NoError(t, err)
} }
@ -545,8 +583,8 @@ func Run(t *testing.T, opt *Opt) {
// TestFsListDirNotFound tests listing the directories from an empty directory // TestFsListDirNotFound tests listing the directories from an empty directory
TestFsListDirNotFound := func(t *testing.T) { TestFsListDirNotFound := func(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
objs, dirs, err := walk.GetAll(ctx, remote, "does not exist", true, 1) objs, dirs, err := walk.GetAll(ctx, f, "does not exist", true, 1)
if !remote.Features().CanHaveEmptyDirectories { if !f.Features().CanHaveEmptyDirectories {
if err != fs.ErrorDirNotFound { if err != fs.ErrorDirNotFound {
assert.NoError(t, err) assert.NoError(t, err)
assert.Equal(t, 0, len(objs)+len(dirs)) assert.Equal(t, 0, len(objs)+len(dirs))
@ -570,7 +608,7 @@ func Run(t *testing.T, opt *Opt) {
skipIfNotOk(t) skipIfNotOk(t)
// check no files or dirs as pre-requisite // check no files or dirs as pre-requisite
fstest.CheckListingWithPrecision(t, remote, []fstest.Item{}, []string{}, fs.GetModifyWindow(ctx, remote)) fstest.CheckListingWithPrecision(t, f, []fstest.Item{}, []string{}, fs.GetModifyWindow(ctx, f))
for _, test := range []struct { for _, test := range []struct {
name string name string
@ -604,16 +642,16 @@ func Run(t *testing.T, opt *Opt) {
fileName := encoder.Standard.Encode(test.path) fileName := encoder.Standard.Encode(test.path)
dirName := fileName dirName := fileName
t.Logf("testing %q", fileName) t.Logf("testing %q", fileName)
assert.NoError(t, remote.Mkdir(ctx, dirName)) assert.NoError(t, f.Mkdir(ctx, dirName))
file := fstest.Item{ file := fstest.Item{
ModTime: time.Now(), ModTime: time.Now(),
Path: dirName + "/" + fileName, // test creating a file and dir with that name Path: dirName + "/" + fileName, // test creating a file and dir with that name
} }
_, o := testPut(context.Background(), t, remote, &file) _, o := testPut(context.Background(), t, f, &file)
fstest.CheckListingWithPrecision(t, remote, []fstest.Item{file}, []string{dirName}, fs.GetModifyWindow(ctx, remote)) fstest.CheckListingWithPrecision(t, f, []fstest.Item{file}, []string{dirName}, fs.GetModifyWindow(ctx, f))
assert.NoError(t, o.Remove(ctx)) assert.NoError(t, o.Remove(ctx))
assert.NoError(t, remote.Rmdir(ctx, dirName)) assert.NoError(t, f.Rmdir(ctx, dirName))
fstest.CheckListingWithPrecision(t, remote, []fstest.Item{}, []string{}, fs.GetModifyWindow(ctx, remote)) fstest.CheckListingWithPrecision(t, f, []fstest.Item{}, []string{}, fs.GetModifyWindow(ctx, f))
}) })
} }
}) })
@ -622,11 +660,11 @@ func Run(t *testing.T, opt *Opt) {
t.Run("FsNewObjectNotFound", func(t *testing.T) { t.Run("FsNewObjectNotFound", func(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
// Object in an existing directory // Object in an existing directory
o, err := remote.NewObject(ctx, "potato") o, err := f.NewObject(ctx, "potato")
assert.Nil(t, o) assert.Nil(t, o)
assert.Equal(t, fs.ErrorObjectNotFound, err) assert.Equal(t, fs.ErrorObjectNotFound, err)
// Now try an object in a non existing directory // Now try an object in a non existing directory
o, err = remote.NewObject(ctx, "directory/not/found/potato") o, err = f.NewObject(ctx, "directory/not/found/potato")
assert.Nil(t, o) assert.Nil(t, o)
assert.Equal(t, fs.ErrorObjectNotFound, err) assert.Equal(t, fs.ErrorObjectNotFound, err)
}) })
@ -653,11 +691,11 @@ func Run(t *testing.T, opt *Opt) {
in := io.MultiReader(buf, er) in := io.MultiReader(buf, er)
obji := object.NewStaticObjectInfo(file2.Path, file2.ModTime, 2*N, true, nil, nil) obji := object.NewStaticObjectInfo(file2.Path, file2.ModTime, 2*N, true, nil, nil)
_, err := remote.Put(ctx, in, obji) _, err := f.Put(ctx, in, obji)
// assert.Nil(t, obj) - FIXME some remotes return the object even on nil // assert.Nil(t, obj) - FIXME some remotes return the object even on nil
assert.NotNil(t, err) assert.NotNil(t, err)
obj, err := remote.NewObject(ctx, file2.Path) obj, err := f.NewObject(ctx, file2.Path)
assert.Nil(t, obj) assert.Nil(t, obj)
assert.Equal(t, fs.ErrorObjectNotFound, err) assert.Equal(t, fs.ErrorObjectNotFound, err)
}) })
@ -665,7 +703,7 @@ func Run(t *testing.T, opt *Opt) {
t.Run("FsPutZeroLength", func(t *testing.T) { t.Run("FsPutZeroLength", func(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
TestPutLarge(ctx, t, remote, &fstest.Item{ TestPutLarge(ctx, t, f, &fstest.Item{
ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"), ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"),
Path: fmt.Sprintf("zero-length-file"), Path: fmt.Sprintf("zero-length-file"),
Size: int64(0), Size: int64(0),
@ -674,7 +712,7 @@ func Run(t *testing.T, opt *Opt) {
t.Run("FsOpenWriterAt", func(t *testing.T) { t.Run("FsOpenWriterAt", func(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
openWriterAt := remote.Features().OpenWriterAt openWriterAt := f.Features().OpenWriterAt
if openWriterAt == nil { if openWriterAt == nil {
t.Skip("FS has no OpenWriterAt interface") t.Skip("FS has no OpenWriterAt interface")
} }
@ -695,11 +733,11 @@ func Run(t *testing.T, opt *Opt) {
assert.NoError(t, out.Close()) assert.NoError(t, out.Close())
obj := findObject(ctx, t, remote, path) obj := findObject(ctx, t, f, path)
assert.Equal(t, "abcdefghi", readObject(ctx, t, obj, -1), "contents of file differ") assert.Equal(t, "abcdefghi", readObject(ctx, t, obj, -1), "contents of file differ")
assert.NoError(t, obj.Remove(ctx)) assert.NoError(t, obj.Remove(ctx))
assert.NoError(t, remote.Rmdir(ctx, "writer-at-subdir")) assert.NoError(t, f.Rmdir(ctx, "writer-at-subdir"))
}) })
// TestFsChangeNotify tests that changes are properly // TestFsChangeNotify tests that changes are properly
@ -710,12 +748,12 @@ func Run(t *testing.T, opt *Opt) {
skipIfNotOk(t) skipIfNotOk(t)
// Check have ChangeNotify // Check have ChangeNotify
doChangeNotify := remote.Features().ChangeNotify doChangeNotify := f.Features().ChangeNotify
if doChangeNotify == nil { if doChangeNotify == nil {
t.Skip("FS has no ChangeNotify interface") t.Skip("FS has no ChangeNotify interface")
} }
err := operations.Mkdir(ctx, remote, "dir") err := operations.Mkdir(ctx, f, "dir")
require.NoError(t, err) require.NoError(t, err)
pollInterval := make(chan time.Duration) pollInterval := make(chan time.Duration)
@ -739,7 +777,7 @@ func Run(t *testing.T, opt *Opt) {
var dirs []string var dirs []string
for _, idx := range []int{1, 3, 2} { for _, idx := range []int{1, 3, 2} {
dir := fmt.Sprintf("dir/subdir%d", idx) dir := fmt.Sprintf("dir/subdir%d", idx)
err = operations.Mkdir(ctx, remote, dir) err = operations.Mkdir(ctx, f, dir)
require.NoError(t, err) require.NoError(t, err)
dirs = append(dirs, dir) dirs = append(dirs, dir)
} }
@ -750,7 +788,7 @@ func Run(t *testing.T, opt *Opt) {
ModTime: time.Now(), ModTime: time.Now(),
Path: fmt.Sprintf("dir/file%d", idx), Path: fmt.Sprintf("dir/file%d", idx),
} }
_, o := testPut(ctx, t, remote, &file) _, o := testPut(ctx, t, f, &file)
objs = append(objs, o) objs = append(objs, o)
} }
@ -788,7 +826,7 @@ func Run(t *testing.T, opt *Opt) {
} }
dirs = append(dirs, "dir") dirs = append(dirs, "dir")
for _, dir := range dirs { for _, dir := range dirs {
assert.NoError(t, remote.Rmdir(ctx, dir)) assert.NoError(t, f.Rmdir(ctx, dir))
} }
}) })
@ -797,9 +835,9 @@ func Run(t *testing.T, opt *Opt) {
// Tests that require file1, file2 are within this // Tests that require file1, file2 are within this
t.Run("FsPutFiles", func(t *testing.T) { t.Run("FsPutFiles", func(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
file1Contents, _ = testPut(ctx, t, remote, &file1) file1Contents, _ = testPut(ctx, t, f, &file1)
/* file2Contents = */ testPut(ctx, t, remote, &file2) /* file2Contents = */ testPut(ctx, t, f, &file2)
file1Contents, _ = testPut(ctx, t, remote, &file1) file1Contents, _ = testPutMimeType(ctx, t, f, &file1, file1MimeType)
// Note that the next test will check there are no duplicated file names // Note that the next test will check there are no duplicated file names
// TestFsListDirFile2 tests the files are correctly uploaded by doing // TestFsListDirFile2 tests the files are correctly uploaded by doing
@ -809,9 +847,9 @@ func Run(t *testing.T, opt *Opt) {
list := func(dir string, expectedDirNames, expectedObjNames []string) { list := func(dir string, expectedDirNames, expectedObjNames []string) {
var objNames, dirNames []string var objNames, dirNames []string
for i := 1; i <= *fstest.ListRetries; i++ { for i := 1; i <= *fstest.ListRetries; i++ {
objs, dirs, err := walk.GetAll(ctx, remote, dir, true, 1) objs, dirs, err := walk.GetAll(ctx, f, dir, true, 1)
if errors.Cause(err) == fs.ErrorDirNotFound { if errors.Cause(err) == fs.ErrorDirNotFound {
objs, dirs, err = walk.GetAll(ctx, remote, dir, true, 1) objs, dirs, err = walk.GetAll(ctx, f, dir, true, 1)
} }
require.NoError(t, err) require.NoError(t, err)
objNames = objsToNames(objs) objNames = objsToNames(objs)
@ -857,7 +895,7 @@ func Run(t *testing.T, opt *Opt) {
// Test the files are all there with walk.ListR recursive listings // Test the files are all there with walk.ListR recursive listings
t.Run("FsListR", func(t *testing.T) { t.Run("FsListR", func(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
objs, dirs, err := walk.GetAll(ctx, remote, "", true, -1) objs, dirs, err := walk.GetAll(ctx, f, "", true, -1)
require.NoError(t, err) require.NoError(t, err)
assert.Equal(t, []string{ assert.Equal(t, []string{
"hello? sausage", "hello? sausage",
@ -875,7 +913,7 @@ func Run(t *testing.T, opt *Opt) {
// walk.ListR recursive listings on a sub dir // walk.ListR recursive listings on a sub dir
t.Run("FsListRSubdir", func(t *testing.T) { t.Run("FsListRSubdir", func(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
objs, dirs, err := walk.GetAll(ctx, remote, path.Dir(path.Dir(path.Dir(path.Dir(file2.Path)))), true, -1) objs, dirs, err := walk.GetAll(ctx, f, path.Dir(path.Dir(path.Dir(path.Dir(file2.Path)))), true, -1)
require.NoError(t, err) require.NoError(t, err)
assert.Equal(t, []string{ assert.Equal(t, []string{
"hello? sausage/êé", "hello? sausage/êé",
@ -914,7 +952,7 @@ func Run(t *testing.T, opt *Opt) {
for i := 0; i < 2; i++ { for i := 0; i < 2; i++ {
dir, _ := path.Split(fileName) dir, _ := path.Split(fileName)
dir = dir[:len(dir)-1] dir = dir[:len(dir)-1]
objs, dirs, err = walk.GetAll(ctx, remote, dir, true, -1) objs, dirs, err = walk.GetAll(ctx, f, dir, true, -1)
} }
require.NoError(t, err) require.NoError(t, err)
require.Len(t, objs, 1) require.Len(t, objs, 1)
@ -932,7 +970,7 @@ func Run(t *testing.T, opt *Opt) {
// TestFsListLevel2 tests List works for 2 levels // TestFsListLevel2 tests List works for 2 levels
TestFsListLevel2 := func(t *testing.T) { TestFsListLevel2 := func(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
objs, dirs, err := walk.GetAll(ctx, remote, "", true, 2) objs, dirs, err := walk.GetAll(ctx, f, "", true, 2)
if err == fs.ErrorLevelNotSupported { if err == fs.ErrorLevelNotSupported {
return return
} }
@ -951,27 +989,27 @@ func Run(t *testing.T, opt *Opt) {
// TestFsListFile1 tests file present // TestFsListFile1 tests file present
t.Run("FsListFile1", func(t *testing.T) { t.Run("FsListFile1", func(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
fstest.CheckListing(t, remote, []fstest.Item{file1, file2}) fstest.CheckListing(t, f, []fstest.Item{file1, file2})
}) })
// TestFsNewObject tests NewObject // TestFsNewObject tests NewObject
t.Run("FsNewObject", func(t *testing.T) { t.Run("FsNewObject", func(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
obj := findObject(ctx, t, remote, file1.Path) obj := findObject(ctx, t, f, file1.Path)
file1.Check(t, obj, remote.Precision()) file1.Check(t, obj, f.Precision())
}) })
// TestFsListFile1and2 tests two files present // TestFsListFile1and2 tests two files present
t.Run("FsListFile1and2", func(t *testing.T) { t.Run("FsListFile1and2", func(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
fstest.CheckListing(t, remote, []fstest.Item{file1, file2}) fstest.CheckListing(t, f, []fstest.Item{file1, file2})
}) })
// TestFsNewObjectDir tests NewObject on a directory which should produce an error // TestFsNewObjectDir tests NewObject on a directory which should produce an error
t.Run("FsNewObjectDir", func(t *testing.T) { t.Run("FsNewObjectDir", func(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
dir := path.Dir(file2.Path) dir := path.Dir(file2.Path)
obj, err := remote.NewObject(ctx, dir) obj, err := f.NewObject(ctx, dir)
assert.Nil(t, obj) assert.Nil(t, obj)
assert.NotNil(t, err) assert.NotNil(t, err)
}) })
@ -981,7 +1019,7 @@ func Run(t *testing.T, opt *Opt) {
skipIfNotOk(t) skipIfNotOk(t)
// Check have Purge // Check have Purge
doPurge := remote.Features().Purge doPurge := f.Features().Purge
if doPurge == nil { if doPurge == nil {
t.Skip("FS has no Purge interface") t.Skip("FS has no Purge interface")
} }
@ -991,26 +1029,26 @@ func Run(t *testing.T, opt *Opt) {
ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"), ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"),
Path: "dirToPurge/fileToPurge.txt", Path: "dirToPurge/fileToPurge.txt",
} }
_, _ = testPut(ctx, t, remote, &fileToPurge) _, _ = testPut(ctx, t, f, &fileToPurge)
fstest.CheckListingWithPrecision(t, remote, []fstest.Item{file1, file2, fileToPurge}, []string{ fstest.CheckListingWithPrecision(t, f, []fstest.Item{file1, file2, fileToPurge}, []string{
"dirToPurge", "dirToPurge",
"hello? sausage", "hello? sausage",
"hello? sausage/êé", "hello? sausage/êé",
"hello? sausage/êé/Hello, 世界", "hello? sausage/êé/Hello, 世界",
"hello? sausage/êé/Hello, 世界/ \" ' @ < > & ? + ≠", "hello? sausage/êé/Hello, 世界/ \" ' @ < > & ? + ≠",
}, fs.GetModifyWindow(ctx, remote)) }, fs.GetModifyWindow(ctx, f))
// Now purge it // Now purge it
err = operations.Purge(ctx, remote, "dirToPurge") err = operations.Purge(ctx, f, "dirToPurge")
require.NoError(t, err) require.NoError(t, err)
fstest.CheckListingWithPrecision(t, remote, []fstest.Item{file1, file2}, []string{ fstest.CheckListingWithPrecision(t, f, []fstest.Item{file1, file2}, []string{
"hello? sausage", "hello? sausage",
"hello? sausage/êé", "hello? sausage/êé",
"hello? sausage/êé/Hello, 世界", "hello? sausage/êé/Hello, 世界",
"hello? sausage/êé/Hello, 世界/ \" ' @ < > & ? + ≠", "hello? sausage/êé/Hello, 世界/ \" ' @ < > & ? + ≠",
}, fs.GetModifyWindow(ctx, remote)) }, fs.GetModifyWindow(ctx, f))
}) })
// TestFsCopy tests Copy // TestFsCopy tests Copy
@ -1018,7 +1056,7 @@ func Run(t *testing.T, opt *Opt) {
skipIfNotOk(t) skipIfNotOk(t)
// Check have Copy // Check have Copy
doCopy := remote.Features().Copy doCopy := f.Features().Copy
if doCopy == nil { if doCopy == nil {
t.Skip("FS has no Copier interface") t.Skip("FS has no Copier interface")
} }
@ -1028,7 +1066,7 @@ func Run(t *testing.T, opt *Opt) {
file2Copy.Path += "-copy" file2Copy.Path += "-copy"
// do the copy // do the copy
src := findObject(ctx, t, remote, file2.Path) src := findObject(ctx, t, f, file2.Path)
dst, err := doCopy(ctx, src, file2Copy.Path) dst, err := doCopy(ctx, src, file2Copy.Path)
if err == fs.ErrorCantCopy { if err == fs.ErrorCantCopy {
t.Skip("FS can't copy") t.Skip("FS can't copy")
@ -1036,7 +1074,7 @@ func Run(t *testing.T, opt *Opt) {
require.NoError(t, err, fmt.Sprintf("Error: %#v", err)) require.NoError(t, err, fmt.Sprintf("Error: %#v", err))
// check file exists in new listing // check file exists in new listing
fstest.CheckListing(t, remote, []fstest.Item{file1, file2, file2Copy}) fstest.CheckListing(t, f, []fstest.Item{file1, file2, file2Copy})
// Check dst lightly - list above has checked ModTime/Hashes // Check dst lightly - list above has checked ModTime/Hashes
assert.Equal(t, file2Copy.Path, dst.Remote()) assert.Equal(t, file2Copy.Path, dst.Remote())
@ -1052,7 +1090,7 @@ func Run(t *testing.T, opt *Opt) {
skipIfNotOk(t) skipIfNotOk(t)
// Check have Move // Check have Move
doMove := remote.Features().Move doMove := f.Features().Move
if doMove == nil { if doMove == nil {
t.Skip("FS has no Mover interface") t.Skip("FS has no Mover interface")
} }
@ -1067,14 +1105,14 @@ func Run(t *testing.T, opt *Opt) {
// check happy path, i.e. no naming conflicts when rename and move are two // check happy path, i.e. no naming conflicts when rename and move are two
// separate operations // separate operations
file2Move.Path = "other.txt" file2Move.Path = "other.txt"
src := findObject(ctx, t, remote, file2.Path) src := findObject(ctx, t, f, file2.Path)
dst, err := doMove(ctx, src, file2Move.Path) dst, err := doMove(ctx, src, file2Move.Path)
if err == fs.ErrorCantMove { if err == fs.ErrorCantMove {
t.Skip("FS can't move") t.Skip("FS can't move")
} }
require.NoError(t, err) require.NoError(t, err)
// check file exists in new listing // check file exists in new listing
fstest.CheckListing(t, remote, []fstest.Item{file1, file2Move}) fstest.CheckListing(t, f, []fstest.Item{file1, file2Move})
// Check dst lightly - list above has checked ModTime/Hashes // Check dst lightly - list above has checked ModTime/Hashes
assert.Equal(t, file2Move.Path, dst.Remote()) assert.Equal(t, file2Move.Path, dst.Remote())
// 1: file name.txt // 1: file name.txt
@ -1082,30 +1120,30 @@ func Run(t *testing.T, opt *Opt) {
// Check conflict on "rename, then move" // Check conflict on "rename, then move"
file1Move.Path = "moveTest/other.txt" file1Move.Path = "moveTest/other.txt"
src = findObject(ctx, t, remote, file1.Path) src = findObject(ctx, t, f, file1.Path)
_, err = doMove(ctx, src, file1Move.Path) _, err = doMove(ctx, src, file1Move.Path)
require.NoError(t, err) require.NoError(t, err)
fstest.CheckListing(t, remote, []fstest.Item{file1Move, file2Move}) fstest.CheckListing(t, f, []fstest.Item{file1Move, file2Move})
// 1: moveTest/other.txt // 1: moveTest/other.txt
// 2: other.txt // 2: other.txt
// Check conflict on "move, then rename" // Check conflict on "move, then rename"
src = findObject(ctx, t, remote, file1Move.Path) src = findObject(ctx, t, f, file1Move.Path)
_, err = doMove(ctx, src, file1.Path) _, err = doMove(ctx, src, file1.Path)
require.NoError(t, err) require.NoError(t, err)
fstest.CheckListing(t, remote, []fstest.Item{file1, file2Move}) fstest.CheckListing(t, f, []fstest.Item{file1, file2Move})
// 1: file name.txt // 1: file name.txt
// 2: other.txt // 2: other.txt
src = findObject(ctx, t, remote, file2Move.Path) src = findObject(ctx, t, f, file2Move.Path)
_, err = doMove(ctx, src, file2.Path) _, err = doMove(ctx, src, file2.Path)
require.NoError(t, err) require.NoError(t, err)
fstest.CheckListing(t, remote, []fstest.Item{file1, file2}) fstest.CheckListing(t, f, []fstest.Item{file1, file2})
// 1: file name.txt // 1: file name.txt
// 2: hello sausage?/../z.txt // 2: hello sausage?/../z.txt
// Tidy up moveTest directory // Tidy up moveTest directory
require.NoError(t, remote.Rmdir(ctx, "moveTest")) require.NoError(t, f.Rmdir(ctx, "moveTest"))
}) })
// Move src to this remote using server-side move operations. // Move src to this remote using server-side move operations.
@ -1123,13 +1161,13 @@ func Run(t *testing.T, opt *Opt) {
skipIfNotOk(t) skipIfNotOk(t)
// Check have DirMove // Check have DirMove
doDirMove := remote.Features().DirMove doDirMove := f.Features().DirMove
if doDirMove == nil { if doDirMove == nil {
t.Skip("FS has no DirMover interface") t.Skip("FS has no DirMover interface")
} }
// Check it can't move onto itself // Check it can't move onto itself
err := doDirMove(ctx, remote, "", "") err := doDirMove(ctx, f, "", "")
require.Equal(t, fs.ErrorDirExists, err) require.Equal(t, fs.ErrorDirExists, err)
// new remote // new remote
@ -1139,12 +1177,12 @@ func Run(t *testing.T, opt *Opt) {
const newName = "new_name/sub_new_name" const newName = "new_name/sub_new_name"
// try the move // try the move
err = newRemote.Features().DirMove(ctx, remote, "", newName) err = newRemote.Features().DirMove(ctx, f, "", newName)
require.NoError(t, err) require.NoError(t, err)
// check remotes // check remotes
// remote should not exist here // remote should not exist here
_, err = remote.List(ctx, "") _, err = f.List(ctx, "")
assert.Equal(t, fs.ErrorDirNotFound, errors.Cause(err)) assert.Equal(t, fs.ErrorDirNotFound, errors.Cause(err))
//fstest.CheckListingWithPrecision(t, remote, []fstest.Item{}, []string{}, remote.Precision()) //fstest.CheckListingWithPrecision(t, remote, []fstest.Item{}, []string{}, remote.Precision())
file1Copy := file1 file1Copy := file1
@ -1165,12 +1203,12 @@ func Run(t *testing.T, opt *Opt) {
require.NoError(t, err) require.NoError(t, err)
// check remotes // check remotes
fstest.CheckListingWithPrecision(t, remote, []fstest.Item{file2, file1}, []string{ fstest.CheckListingWithPrecision(t, f, []fstest.Item{file2, file1}, []string{
"hello? sausage", "hello? sausage",
"hello? sausage/êé", "hello? sausage/êé",
"hello? sausage/êé/Hello, 世界", "hello? sausage/êé/Hello, 世界",
"hello? sausage/êé/Hello, 世界/ \" ' @ < > & ? + ≠", "hello? sausage/êé/Hello, 世界/ \" ' @ < > & ? + ≠",
}, remote.Precision()) }, f.Precision())
fstest.CheckListingWithPrecision(t, newRemote, []fstest.Item{}, []string{ fstest.CheckListingWithPrecision(t, newRemote, []fstest.Item{}, []string{
"new_name", "new_name",
}, newRemote.Precision()) }, newRemote.Precision())
@ -1179,17 +1217,17 @@ func Run(t *testing.T, opt *Opt) {
// TestFsRmdirFull tests removing a non empty directory // TestFsRmdirFull tests removing a non empty directory
t.Run("FsRmdirFull", func(t *testing.T) { t.Run("FsRmdirFull", func(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
if isBucketBasedButNotRoot(remote) { if isBucketBasedButNotRoot(f) {
t.Skip("Skipping test as non root bucket based remote") t.Skip("Skipping test as non root bucket based remote")
} }
err := remote.Rmdir(ctx, "") err := f.Rmdir(ctx, "")
require.Error(t, err, "Expecting error on RMdir on non empty remote") require.Error(t, err, "Expecting error on RMdir on non empty remote")
}) })
// TestFsPrecision tests the Precision of the Fs // TestFsPrecision tests the Precision of the Fs
t.Run("FsPrecision", func(t *testing.T) { t.Run("FsPrecision", func(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
precision := remote.Precision() precision := f.Precision()
if precision == fs.ModTimeNotSupported { if precision == fs.ModTimeNotSupported {
return return
} }
@ -1202,7 +1240,7 @@ func Run(t *testing.T, opt *Opt) {
// TestObjectString tests the Object String method // TestObjectString tests the Object String method
t.Run("ObjectString", func(t *testing.T) { t.Run("ObjectString", func(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
obj := findObject(ctx, t, remote, file1.Path) obj := findObject(ctx, t, f, file1.Path)
assert.Equal(t, file1.Path, obj.String()) assert.Equal(t, file1.Path, obj.String())
if opt.NilObject != nil { if opt.NilObject != nil {
assert.Equal(t, "<nil>", opt.NilObject.String()) assert.Equal(t, "<nil>", opt.NilObject.String())
@ -1212,13 +1250,13 @@ func Run(t *testing.T, opt *Opt) {
// TestObjectFs tests the object can be found // TestObjectFs tests the object can be found
t.Run("ObjectFs", func(t *testing.T) { t.Run("ObjectFs", func(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
obj := findObject(ctx, t, remote, file1.Path) obj := findObject(ctx, t, f, file1.Path)
// If this is set we don't do the direct comparison of // If this is set we don't do the direct comparison of
// the Fs from the object as it may be different // the Fs from the object as it may be different
if opt.SkipFsMatch { if opt.SkipFsMatch {
return return
} }
testRemote := remote testRemote := f
if obj.Fs() != testRemote { if obj.Fs() != testRemote {
// Check to see if this wraps something else // Check to see if this wraps something else
if doUnWrap := testRemote.Features().UnWrap; doUnWrap != nil { if doUnWrap := testRemote.Features().UnWrap; doUnWrap != nil {
@ -1231,38 +1269,44 @@ func Run(t *testing.T, opt *Opt) {
// TestObjectRemote tests the Remote is correct // TestObjectRemote tests the Remote is correct
t.Run("ObjectRemote", func(t *testing.T) { t.Run("ObjectRemote", func(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
obj := findObject(ctx, t, remote, file1.Path) obj := findObject(ctx, t, f, file1.Path)
assert.Equal(t, file1.Path, obj.Remote()) assert.Equal(t, file1.Path, obj.Remote())
}) })
// TestObjectHashes checks all the hashes the object supports // TestObjectHashes checks all the hashes the object supports
t.Run("ObjectHashes", func(t *testing.T) { t.Run("ObjectHashes", func(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
obj := findObject(ctx, t, remote, file1.Path) obj := findObject(ctx, t, f, file1.Path)
file1.CheckHashes(t, obj) file1.CheckHashes(t, obj)
}) })
// TestObjectModTime tests the ModTime of the object is correct // TestObjectModTime tests the ModTime of the object is correct
TestObjectModTime := func(t *testing.T) { TestObjectModTime := func(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
obj := findObject(ctx, t, remote, file1.Path) obj := findObject(ctx, t, f, file1.Path)
file1.CheckModTime(t, obj, obj.ModTime(ctx), remote.Precision()) file1.CheckModTime(t, obj, obj.ModTime(ctx), f.Precision())
} }
t.Run("ObjectModTime", TestObjectModTime) t.Run("ObjectModTime", TestObjectModTime)
// TestObjectMimeType tests the MimeType of the object is correct // TestObjectMimeType tests the MimeType of the object is correct
t.Run("ObjectMimeType", func(t *testing.T) { t.Run("ObjectMimeType", func(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
obj := findObject(ctx, t, remote, file1.Path) features := f.Features()
obj := findObject(ctx, t, f, file1.Path)
do, ok := obj.(fs.MimeTyper) do, ok := obj.(fs.MimeTyper)
require.Equal(t, features.ReadMimeType, ok, "mismatch between Object.MimeType and Features.ReadMimeType")
if !ok { if !ok {
t.Skip("MimeType method not supported") t.Skip("MimeType method not supported")
} }
mimeType := do.MimeType(ctx) mimeType := do.MimeType(ctx)
if strings.ContainsRune(mimeType, ';') { if features.WriteMimeType {
assert.Equal(t, "text/plain; charset=utf-8", mimeType) assert.Equal(t, file1MimeType, mimeType, "can read and write mime types but failed")
} else { } else {
assert.Equal(t, "text/plain", mimeType) if strings.ContainsRune(mimeType, ';') {
assert.Equal(t, "text/plain; charset=utf-8", mimeType)
} else {
assert.Equal(t, "text/plain", mimeType)
}
} }
}) })
@ -1270,7 +1314,7 @@ func Run(t *testing.T, opt *Opt) {
t.Run("ObjectSetModTime", func(t *testing.T) { t.Run("ObjectSetModTime", func(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
newModTime := fstest.Time("2011-12-13T14:15:16.999999999Z") newModTime := fstest.Time("2011-12-13T14:15:16.999999999Z")
obj := findObject(ctx, t, remote, file1.Path) obj := findObject(ctx, t, f, file1.Path)
err := obj.SetModTime(ctx, newModTime) err := obj.SetModTime(ctx, newModTime)
if err == fs.ErrorCantSetModTime || err == fs.ErrorCantSetModTimeWithoutDelete { if err == fs.ErrorCantSetModTime || err == fs.ErrorCantSetModTimeWithoutDelete {
t.Log(err) t.Log(err)
@ -1278,7 +1322,7 @@ func Run(t *testing.T, opt *Opt) {
} }
require.NoError(t, err) require.NoError(t, err)
file1.ModTime = newModTime file1.ModTime = newModTime
file1.CheckModTime(t, obj, obj.ModTime(ctx), remote.Precision()) file1.CheckModTime(t, obj, obj.ModTime(ctx), f.Precision())
// And make a new object and read it from there too // And make a new object and read it from there too
TestObjectModTime(t) TestObjectModTime(t)
}) })
@ -1286,21 +1330,21 @@ func Run(t *testing.T, opt *Opt) {
// TestObjectSize tests that Size works // TestObjectSize tests that Size works
t.Run("ObjectSize", func(t *testing.T) { t.Run("ObjectSize", func(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
obj := findObject(ctx, t, remote, file1.Path) obj := findObject(ctx, t, f, file1.Path)
assert.Equal(t, file1.Size, obj.Size()) assert.Equal(t, file1.Size, obj.Size())
}) })
// TestObjectOpen tests that Open works // TestObjectOpen tests that Open works
t.Run("ObjectOpen", func(t *testing.T) { t.Run("ObjectOpen", func(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
obj := findObject(ctx, t, remote, file1.Path) obj := findObject(ctx, t, f, file1.Path)
assert.Equal(t, file1Contents, readObject(ctx, t, obj, -1), "contents of file1 differ") assert.Equal(t, file1Contents, readObject(ctx, t, obj, -1), "contents of file1 differ")
}) })
// TestObjectOpenSeek tests that Open works with SeekOption // TestObjectOpenSeek tests that Open works with SeekOption
t.Run("ObjectOpenSeek", func(t *testing.T) { t.Run("ObjectOpenSeek", func(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
obj := findObject(ctx, t, remote, file1.Path) obj := findObject(ctx, t, f, file1.Path)
assert.Equal(t, file1Contents[50:], readObject(ctx, t, obj, -1, &fs.SeekOption{Offset: 50}), "contents of file1 differ after seek") assert.Equal(t, file1Contents[50:], readObject(ctx, t, obj, -1, &fs.SeekOption{Offset: 50}), "contents of file1 differ after seek")
}) })
@ -1309,7 +1353,7 @@ func Run(t *testing.T, opt *Opt) {
// go test -v -run 'TestIntegration/Test(Setup|Init|FsMkdir|FsPutFile1|FsPutFile2|FsUpdateFile1|ObjectOpenRange)$' // go test -v -run 'TestIntegration/Test(Setup|Init|FsMkdir|FsPutFile1|FsPutFile2|FsUpdateFile1|ObjectOpenRange)$'
t.Run("ObjectOpenRange", func(t *testing.T) { t.Run("ObjectOpenRange", func(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
obj := findObject(ctx, t, remote, file1.Path) obj := findObject(ctx, t, f, file1.Path)
for _, test := range []struct { for _, test := range []struct {
ro fs.RangeOption ro fs.RangeOption
wantStart, wantEnd int wantStart, wantEnd int
@ -1330,7 +1374,7 @@ func Run(t *testing.T, opt *Opt) {
// TestObjectPartialRead tests that reading only part of the object does the correct thing // TestObjectPartialRead tests that reading only part of the object does the correct thing
t.Run("ObjectPartialRead", func(t *testing.T) { t.Run("ObjectPartialRead", func(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
obj := findObject(ctx, t, remote, file1.Path) obj := findObject(ctx, t, f, file1.Path)
assert.Equal(t, file1Contents[:50], readObject(ctx, t, obj, 50), "contents of file1 differ after limited read") assert.Equal(t, file1Contents[:50], readObject(ctx, t, obj, 50), "contents of file1 differ after limited read")
}) })
@ -1343,18 +1387,18 @@ func Run(t *testing.T, opt *Opt) {
in := io.TeeReader(buf, hash) in := io.TeeReader(buf, hash)
file1.Size = int64(buf.Len()) file1.Size = int64(buf.Len())
obj := findObject(ctx, t, remote, file1.Path) obj := findObject(ctx, t, f, file1.Path)
obji := object.NewStaticObjectInfo(file1.Path, file1.ModTime, int64(len(contents)), true, nil, obj.Fs()) obji := object.NewStaticObjectInfo(file1.Path, file1.ModTime, int64(len(contents)), true, nil, obj.Fs())
err := obj.Update(ctx, in, obji) err := obj.Update(ctx, in, obji)
require.NoError(t, err) require.NoError(t, err)
file1.Hashes = hash.Sums() file1.Hashes = hash.Sums()
// check the object has been updated // check the object has been updated
file1.Check(t, obj, remote.Precision()) file1.Check(t, obj, f.Precision())
// Re-read the object and check again // Re-read the object and check again
obj = findObject(ctx, t, remote, file1.Path) obj = findObject(ctx, t, f, file1.Path)
file1.Check(t, obj, remote.Precision()) file1.Check(t, obj, f.Precision())
// check contents correct // check contents correct
assert.Equal(t, contents, readObject(ctx, t, obj, -1), "contents of updated file1 differ") assert.Equal(t, contents, readObject(ctx, t, obj, -1), "contents of updated file1 differ")
@ -1364,7 +1408,7 @@ func Run(t *testing.T, opt *Opt) {
// TestObjectStorable tests that Storable works // TestObjectStorable tests that Storable works
t.Run("ObjectStorable", func(t *testing.T) { t.Run("ObjectStorable", func(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
obj := findObject(ctx, t, remote, file1.Path) obj := findObject(ctx, t, f, file1.Path)
require.NotNil(t, !obj.Storable(), "Expecting object to be storable") require.NotNil(t, !obj.Storable(), "Expecting object to be storable")
}) })
@ -1398,7 +1442,7 @@ func Run(t *testing.T, opt *Opt) {
// Test that things work from the root // Test that things work from the root
t.Run("FromRoot", func(t *testing.T) { t.Run("FromRoot", func(t *testing.T) {
if features := remote.Features(); features.BucketBased && !features.BucketBasedRootOK { if features := f.Features(); features.BucketBased && !features.BucketBasedRootOK {
t.Skip("Can't list from root on this remote") t.Skip("Can't list from root on this remote")
} }
@ -1500,7 +1544,7 @@ func Run(t *testing.T, opt *Opt) {
t.Run("PublicLink", func(t *testing.T) { t.Run("PublicLink", func(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
doPublicLink := remote.Features().PublicLink doPublicLink := f.Features().PublicLink
if doPublicLink == nil { if doPublicLink == nil {
t.Skip("FS has no PublicLinker interface") t.Skip("FS has no PublicLinker interface")
} }
@ -1561,7 +1605,7 @@ func Run(t *testing.T, opt *Opt) {
// TestSetTier tests SetTier and GetTier functionality // TestSetTier tests SetTier and GetTier functionality
t.Run("SetTier", func(t *testing.T) { t.Run("SetTier", func(t *testing.T) {
skipIfNotSetTier(t) skipIfNotSetTier(t)
obj := findObject(ctx, t, remote, file1.Path) obj := findObject(ctx, t, f, file1.Path)
setter, ok := obj.(fs.SetTierer) setter, ok := obj.(fs.SetTierer)
assert.NotNil(t, ok) assert.NotNil(t, ok)
getter, ok := obj.(fs.GetTierer) getter, ok := obj.(fs.GetTierer)
@ -1585,11 +1629,11 @@ func Run(t *testing.T, opt *Opt) {
if opt.SkipObjectCheckWrap { if opt.SkipObjectCheckWrap {
t.Skip("Skipping FsCheckWrap on this Fs") t.Skip("Skipping FsCheckWrap on this Fs")
} }
ft := new(fs.Features).Fill(ctx, remote) ft := new(fs.Features).Fill(ctx, f)
if ft.UnWrap == nil { if ft.UnWrap == nil {
t.Skip("Not a wrapping Fs") t.Skip("Not a wrapping Fs")
} }
obj := findObject(ctx, t, remote, file1.Path) obj := findObject(ctx, t, f, file1.Path)
_, unsupported := fs.ObjectOptionalInterfaces(obj) _, unsupported := fs.ObjectOptionalInterfaces(obj)
for _, name := range unsupported { for _, name := range unsupported {
if !stringsContains(name, opt.UnimplementableObjectMethods) { if !stringsContains(name, opt.UnimplementableObjectMethods) {
@ -1602,11 +1646,11 @@ func Run(t *testing.T, opt *Opt) {
t.Run("ObjectRemove", func(t *testing.T) { t.Run("ObjectRemove", func(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
// remove file1 // remove file1
obj := findObject(ctx, t, remote, file1.Path) obj := findObject(ctx, t, f, file1.Path)
err := obj.Remove(ctx) err := obj.Remove(ctx)
require.NoError(t, err) require.NoError(t, err)
// check listing without modtime as TestPublicLink may change the modtime // check listing without modtime as TestPublicLink may change the modtime
fstest.CheckListingWithPrecision(t, remote, []fstest.Item{file2}, nil, fs.ModTimeNotSupported) fstest.CheckListingWithPrecision(t, f, []fstest.Item{file2}, nil, fs.ModTimeNotSupported)
}) })
// TestAbout tests the About optional interface // TestAbout tests the About optional interface
@ -1614,7 +1658,7 @@ func Run(t *testing.T, opt *Opt) {
skipIfNotOk(t) skipIfNotOk(t)
// Check have About // Check have About
doAbout := remote.Features().About doAbout := f.Features().About
if doAbout == nil { if doAbout == nil {
t.Skip("FS does not support About") t.Skip("FS does not support About")
} }
@ -1633,7 +1677,7 @@ func Run(t *testing.T, opt *Opt) {
// close to the end of suite. (See fs/operations/xtra_operations_test.go) // close to the end of suite. (See fs/operations/xtra_operations_test.go)
t.Run("FsPutStream", func(t *testing.T) { t.Run("FsPutStream", func(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
if remote.Features().PutStream == nil { if f.Features().PutStream == nil {
t.Skip("FS has no PutStream interface") t.Skip("FS has no PutStream interface")
} }
@ -1658,15 +1702,15 @@ func Run(t *testing.T, opt *Opt) {
file.Size = -1 file.Size = -1
obji := object.NewStaticObjectInfo(file.Path, file.ModTime, file.Size, true, nil, nil) obji := object.NewStaticObjectInfo(file.Path, file.ModTime, file.Size, true, nil, nil)
obj, err = remote.Features().PutStream(ctx, in, obji) obj, err = f.Features().PutStream(ctx, in, obji)
return err return err
}) })
file.Hashes = uploadHash.Sums() file.Hashes = uploadHash.Sums()
file.Size = int64(contentSize) // use correct size when checking file.Size = int64(contentSize) // use correct size when checking
file.Check(t, obj, remote.Precision()) file.Check(t, obj, f.Precision())
// Re-read the object and check again // Re-read the object and check again
obj = findObject(ctx, t, remote, file.Path) obj = findObject(ctx, t, f, file.Path)
file.Check(t, obj, remote.Precision()) file.Check(t, obj, f.Precision())
require.NoError(t, obj.Remove(ctx)) require.NoError(t, obj.Remove(ctx))
}) })
} }
@ -1675,10 +1719,10 @@ func Run(t *testing.T, opt *Opt) {
// TestInternal calls InternalTest() on the Fs // TestInternal calls InternalTest() on the Fs
t.Run("Internal", func(t *testing.T) { t.Run("Internal", func(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
if it, ok := remote.(InternalTester); ok { if it, ok := f.(InternalTester); ok {
it.InternalTest(t) it.InternalTest(t)
} else { } else {
t.Skipf("%T does not implement InternalTester", remote) t.Skipf("%T does not implement InternalTester", f)
} }
}) })
@ -1693,12 +1737,12 @@ func Run(t *testing.T, opt *Opt) {
t.Skip("not running with -short") t.Skip("not running with -short")
} }
setUploadChunkSizer, _ := remote.(SetUploadChunkSizer) setUploadChunkSizer, _ := f.(SetUploadChunkSizer)
if setUploadChunkSizer == nil { if setUploadChunkSizer == nil {
t.Skipf("%T does not implement SetUploadChunkSizer", remote) t.Skipf("%T does not implement SetUploadChunkSizer", f)
} }
setUploadCutoffer, _ := remote.(SetUploadCutoffer) setUploadCutoffer, _ := f.(SetUploadCutoffer)
minChunkSize := opt.ChunkedUpload.MinChunkSize minChunkSize := opt.ChunkedUpload.MinChunkSize
if minChunkSize < 100 { if minChunkSize < 100 {
@ -1782,7 +1826,7 @@ func Run(t *testing.T, opt *Opt) {
for _, fileSize := range testChunks { for _, fileSize := range testChunks {
t.Run(fmt.Sprintf("%d", fileSize), func(t *testing.T) { t.Run(fmt.Sprintf("%d", fileSize), func(t *testing.T) {
TestPutLarge(ctx, t, remote, &fstest.Item{ TestPutLarge(ctx, t, f, &fstest.Item{
ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"), ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"),
Path: fmt.Sprintf("chunked-%s-%s.bin", cs.String(), fileSize.String()), Path: fmt.Sprintf("chunked-%s-%s.bin", cs.String(), fileSize.String()),
Size: int64(fileSize), Size: int64(fileSize),
@ -1810,7 +1854,7 @@ func Run(t *testing.T, opt *Opt) {
in := bytes.NewBufferString(contents) in := bytes.NewBufferString(contents)
obji := object.NewStaticObjectInfo("unknown-size-put.txt", fstest.Time("2002-02-03T04:05:06.499999999Z"), -1, true, nil, nil) obji := object.NewStaticObjectInfo("unknown-size-put.txt", fstest.Time("2002-02-03T04:05:06.499999999Z"), -1, true, nil, nil)
obj, err := remote.Put(ctx, in, obji) obj, err := f.Put(ctx, in, obji)
if err == nil { if err == nil {
require.NoError(t, obj.Remove(ctx), "successfully uploaded unknown-sized file but failed to remove") require.NoError(t, obj.Remove(ctx), "successfully uploaded unknown-sized file but failed to remove")
} }
@ -1823,7 +1867,7 @@ func Run(t *testing.T, opt *Opt) {
Path: "unknown-size-update.txt", Path: "unknown-size-update.txt",
} }
testPut(ctx, t, remote, &unknownSizeUpdateFile) testPut(ctx, t, f, &unknownSizeUpdateFile)
defer func() { defer func() {
assert.Nil(t, recover(), "Object.Update() should not panic when src.Size() == -1") assert.Nil(t, recover(), "Object.Update() should not panic when src.Size() == -1")
@ -1832,7 +1876,7 @@ func Run(t *testing.T, opt *Opt) {
newContents := random.String(200) newContents := random.String(200)
in := bytes.NewBufferString(newContents) in := bytes.NewBufferString(newContents)
obj := findObject(ctx, t, remote, unknownSizeUpdateFile.Path) obj := findObject(ctx, t, f, unknownSizeUpdateFile.Path)
obji := object.NewStaticObjectInfo(unknownSizeUpdateFile.Path, unknownSizeUpdateFile.ModTime, -1, true, nil, obj.Fs()) obji := object.NewStaticObjectInfo(unknownSizeUpdateFile.Path, unknownSizeUpdateFile.ModTime, -1, true, nil, obj.Fs())
err := obj.Update(ctx, in, obji) err := obj.Update(ctx, in, obji)
if err == nil { if err == nil {
@ -1865,16 +1909,16 @@ func Run(t *testing.T, opt *Opt) {
}) })
// Purge the folder // Purge the folder
err = operations.Purge(ctx, remote, "") err = operations.Purge(ctx, f, "")
if errors.Cause(err) != fs.ErrorDirNotFound { if errors.Cause(err) != fs.ErrorDirNotFound {
require.NoError(t, err) require.NoError(t, err)
} }
purged = true purged = true
fstest.CheckListing(t, remote, []fstest.Item{}) fstest.CheckListing(t, f, []fstest.Item{})
// Check purging again if not bucket based // Check purging again if not bucket based
if !isBucketBasedButNotRoot(remote) { if !isBucketBasedButNotRoot(f) {
err = operations.Purge(ctx, remote, "") err = operations.Purge(ctx, f, "")
assert.Error(t, err, "Expecting error after on second purge") assert.Error(t, err, "Expecting error after on second purge")
if errors.Cause(err) != fs.ErrorDirNotFound { if errors.Cause(err) != fs.ErrorDirNotFound {
t.Log("Warning: this should produce fs.ErrorDirNotFound") t.Log("Warning: this should produce fs.ErrorDirNotFound")
@ -1885,7 +1929,7 @@ func Run(t *testing.T, opt *Opt) {
// Check directory is purged // Check directory is purged
if !purged { if !purged {
_ = operations.Purge(ctx, remote, "") _ = operations.Purge(ctx, f, "")
} }
// Remove the local directory so we don't clutter up /tmp // Remove the local directory so we don't clutter up /tmp