diff --git a/backend/cache/cache_test.go b/backend/cache/cache_test.go index 547861234..e0df97bf3 100644 --- a/backend/cache/cache_test.go +++ b/backend/cache/cache_test.go @@ -17,7 +17,7 @@ func TestIntegration(t *testing.T) { fstests.Run(t, &fstests.Opt{ RemoteName: "TestCache:", NilObject: (*cache.Object)(nil), - UnimplementableFsMethods: []string{"PublicLink", "MergeDirs"}, + UnimplementableFsMethods: []string{"PublicLink", "MergeDirs", "OpenWriterAt"}, UnimplementableObjectMethods: []string{"MimeType", "ID", "GetTier", "SetTier"}, }) } diff --git a/backend/crypt/crypt_test.go b/backend/crypt/crypt_test.go index 1558915c4..72073400d 100644 --- a/backend/crypt/crypt_test.go +++ b/backend/crypt/crypt_test.go @@ -23,6 +23,7 @@ func TestIntegration(t *testing.T) { fstests.Run(t, &fstests.Opt{ RemoteName: *fstest.RemoteName, NilObject: (*crypt.Object)(nil), + UnimplementableFsMethods: []string{"OpenWriterAt"}, UnimplementableObjectMethods: []string{"MimeType"}, }) } @@ -43,6 +44,7 @@ func TestStandard(t *testing.T) { {Name: name, Key: "password", Value: obscure.MustObscure("potato")}, {Name: name, Key: "filename_encryption", Value: "standard"}, }, + UnimplementableFsMethods: []string{"OpenWriterAt"}, UnimplementableObjectMethods: []string{"MimeType"}, }) } @@ -63,6 +65,7 @@ func TestOff(t *testing.T) { {Name: name, Key: "password", Value: obscure.MustObscure("potato2")}, {Name: name, Key: "filename_encryption", Value: "off"}, }, + UnimplementableFsMethods: []string{"OpenWriterAt"}, UnimplementableObjectMethods: []string{"MimeType"}, }) } @@ -84,6 +87,7 @@ func TestObfuscate(t *testing.T) { {Name: name, Key: "filename_encryption", Value: "obfuscate"}, }, SkipBadWindowsCharacters: true, + UnimplementableFsMethods: []string{"OpenWriterAt"}, UnimplementableObjectMethods: []string{"MimeType"}, }) } diff --git a/backend/local/local.go b/backend/local/local.go index 409b2d48c..4506a3118 100644 --- a/backend/local/local.go +++ b/backend/local/local.go @@ -998,6 +998,36 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio return o.lstat() } +// OpenWriterAt opens with a handle for random access writes +// +// Pass in the remote desired and the size if known. +// +// It truncates any existing object +func (f *Fs) OpenWriterAt(remote string, size int64) (fs.WriterAtCloser, error) { + // Temporary Object under construction + o := f.newObject(remote, "") + + err := o.mkdirAll() + if err != nil { + return nil, err + } + + if o.translatedLink { + return nil, errors.New("can't open a symlink for random writing") + } + + out, err := file.OpenFile(o.path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666) + if err != nil { + return nil, err + } + // Pre-allocate the file for performance reasons + err = preAllocate(size, out) + if err != nil { + fs.Debugf(o, "Failed to pre-allocate: %v", err) + } + return out, nil +} + // setMetadata sets the file info from the os.FileInfo passed in func (o *Object) setMetadata(info os.FileInfo) { // Don't overwrite the info if we don't need to @@ -1139,10 +1169,11 @@ func cleanWindowsName(f *Fs, name string) string { // Check the interfaces are satisfied var ( - _ fs.Fs = &Fs{} - _ fs.Purger = &Fs{} - _ fs.PutStreamer = &Fs{} - _ fs.Mover = &Fs{} - _ fs.DirMover = &Fs{} - _ fs.Object = &Object{} + _ fs.Fs = &Fs{} + _ fs.Purger = &Fs{} + _ fs.PutStreamer = &Fs{} + _ fs.Mover = &Fs{} + _ fs.DirMover = &Fs{} + _ fs.OpenWriterAter = &Fs{} + _ fs.Object = &Object{} ) diff --git a/fs/fs.go b/fs/fs.go index dc31fdce9..1617cf9ee 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -427,6 +427,12 @@ type Usage struct { Objects *int64 `json:"objects,omitempty"` // objects in the storage system } +// WriterAtCloser wraps io.WriterAt and io.Closer +type WriterAtCloser interface { + io.WriterAt + io.Closer +} + // Features describe the optional features of the Fs type Features struct { // Feature flags, whether Fs @@ -548,6 +554,13 @@ type Features struct { // About gets quota information from the Fs About func() (*Usage, error) + + // OpenWriterAt opens with a handle for random access writes + // + // Pass in the remote desired and the size if known. + // + // It truncates any existing object + OpenWriterAt func(remote string, size int64) (WriterAtCloser, error) } // Disable nil's out the named feature. If it isn't found then it @@ -640,6 +653,9 @@ func (ft *Features) Fill(f Fs) *Features { if do, ok := f.(Abouter); ok { ft.About = do.About } + if do, ok := f.(OpenWriterAter); ok { + ft.OpenWriterAt = do.OpenWriterAt + } return ft.DisableList(Config.DisableFeatures) } @@ -705,6 +721,9 @@ func (ft *Features) Mask(f Fs) *Features { if mask.About == nil { ft.About = nil } + if mask.OpenWriterAt == nil { + ft.OpenWriterAt = nil + } return ft.DisableList(Config.DisableFeatures) } @@ -904,6 +923,16 @@ type Abouter interface { About() (*Usage, error) } +// OpenWriterAter is an optional interface for Fs +type OpenWriterAter interface { + // OpenWriterAt opens with a handle for random access writes + // + // Pass in the remote desired and the size if known. + // + // It truncates any existing object + OpenWriterAt(remote string, size int64) (WriterAtCloser, error) +} + // ObjectsChan is a channel of Objects type ObjectsChan chan Object diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go index 82c1f7f77..86384c371 100644 --- a/fstest/fstests/fstests.go +++ b/fstest/fstests/fstests.go @@ -674,6 +674,36 @@ func Run(t *testing.T, opt *Opt) { } }) + t.Run("FsOpenWriterAt", func(t *testing.T) { + skipIfNotOk(t) + openWriterAt := remote.Features().OpenWriterAt + if openWriterAt == nil { + t.Skip("FS has no OpenWriterAt interface") + } + path := "writer-at-subdir/writer-at-file" + out, err := openWriterAt(path, -1) + require.NoError(t, err) + + var n int + n, err = out.WriteAt([]byte("def"), 3) + assert.NoError(t, err) + assert.Equal(t, 3, n) + n, err = out.WriteAt([]byte("ghi"), 6) + assert.NoError(t, err) + assert.Equal(t, 3, n) + n, err = out.WriteAt([]byte("abc"), 0) + assert.NoError(t, err) + assert.Equal(t, 3, n) + + assert.NoError(t, out.Close()) + + obj := findObject(t, remote, path) + assert.Equal(t, "abcdefghi", readObject(t, obj, -1), "contents of file differ") + + assert.NoError(t, obj.Remove()) + assert.NoError(t, remote.Rmdir("writer-at-subdir")) + }) + // TestFsChangeNotify tests that changes are properly // propagated //