community
/
rclone
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

s3: add --s3-version-deleted to show delete markers in listings when using versions. 

See: https://forum.rclone.org/t/s3-object-deletion-times/42781
This commit is contained in:
Nick Craig-Wood 2023-11-08 15:29:23 +00:00
parent 9bfbf2a4ae
commit 4d4f3de5a5
1 changed files with 23 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
backend/s3/s3.go
View File

@ -2426,6 +2426,19 @@ See [the time option docs](/docs/#time-option) for valid formats.
`,
Default: fs.Time{},
Advanced: true,
}, {
Name: "version_deleted",
Help: `Show deleted file markers when using versions.
This shows deleted file markers in the listing when using versions. These will appear
as 0 size files. The only operation which can be performed on them is deletion.
Deleting a delete marker will reveal the previous version.
Deleted files will always show with a timestamp.
`,
Default: false,
Advanced: true,
}, {
Name: "decompress",
Help: `If set this will decompress gzip encoded objects.
@ -2653,6 +2666,7 @@ type Options struct {
UsePresignedRequest bool `config:"use_presigned_request"`
Versions bool `config:"versions"`
VersionAt fs.Time `config:"version_at"`
VersionDeleted bool `config:"version_deleted"`
Decompress bool `config:"decompress"`
MightGzip fs.Tristate `config:"might_gzip"`
UseAcceptEncodingGzip fs.Tristate `config:"use_accept_encoding_gzip"`
@ -3420,6 +3434,7 @@ func (f *Fs) getMetaDataListing(ctx context.Context, wantRemote string) (info *s
withVersions: f.opt.Versions,
findFile: true,
versionAt: f.opt.VersionAt,
hidden: f.opt.VersionDeleted,
}, func(gotRemote string, object *s3.Object, objectVersionID *string, isDirectory bool) error {
if isDirectory {
return nil
@ -3481,6 +3496,10 @@ func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *s3.Obje
o.bytes = aws.Int64Value(info.Size)
o.storageClass = stringClonePointer(info.StorageClass)
o.versionID = stringClonePointer(versionID)
// If is delete marker, show that metadata has been read as there is none to read
if info.Size == isDeleteMarker {
o.meta = map[string]string{}
}
} else if !o.fs.opt.NoHeadObject {
err := o.readMetaData(ctx) // reads info and meta, returning an error
if err != nil {
@ -3778,7 +3797,7 @@ func (ls *versionsList) List(ctx context.Context) (resp *s3.ListObjectsV2Output,
//structs.SetFrom(obj, objVersion)
setFrom_s3Object_s3ObjectVersion(obj, objVersion)
// Adjust the file names
if !ls.usingVersionAt && !aws.BoolValue(objVersion.IsLatest) {
if !ls.usingVersionAt && (!aws.BoolValue(objVersion.IsLatest) || objVersion.Size == isDeleteMarker) {
if obj.Key != nil && objVersion.LastModified != nil {
*obj.Key = version.Add(*obj.Key, *objVersion.LastModified)
}
@ -4046,6 +4065,7 @@ func (f *Fs) listDir(ctx context.Context, bucket, directory, prefix string, addB
addBucket: addBucket,
withVersions: f.opt.Versions,
versionAt: f.opt.VersionAt,
hidden: f.opt.VersionDeleted,
}, func(remote string, object *s3.Object, versionID *string, isDirectory bool) error {
entry, err := f.itemToDirEntry(ctx, remote, object, versionID, isDirectory)
if err != nil {
@ -4132,6 +4152,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
recurse: true,
withVersions: f.opt.Versions,
versionAt: f.opt.VersionAt,
hidden: f.opt.VersionDeleted,
}, func(remote string, object *s3.Object, versionID *string, isDirectory bool) error {
entry, err := f.itemToDirEntry(ctx, remote, object, versionID, isDirectory)
if err != nil {
@ -4898,6 +4919,7 @@ func (f *Fs) restoreStatus(ctx context.Context, all bool) (out []restoreStatusOu
recurse: true,
withVersions: f.opt.Versions,
versionAt: f.opt.VersionAt,
hidden: f.opt.VersionDeleted,
restoreStatus: true,
}, func(remote string, object *s3.Object, versionID *string, isDirectory bool) error {
entry, err := f.itemToDirEntry(ctx, remote, object, versionID, isDirectory)