From 0652ec95db726f8895cb4aa402df04417439c270 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 22 Jun 2022 15:56:41 +0100 Subject: [PATCH] fs: implement MetadataInfo to show info about metadata in help and rc Info about this will appear in operations/fsinfo and in the backend help (`rclone help backend s3`). --- cmd/help.go | 25 +++++++ fs/metadata.go | 14 ++++ fs/operations/operations.go | 21 ++++-- fs/operations/rc.go | 137 +++++++++++++++++++++++++----------- fs/registry.go | 2 + fstest/fstests/fstests.go | 9 ++- 6 files changed, 160 insertions(+), 48 deletions(-) diff --git a/cmd/help.go b/cmd/help.go index db7e95da4..b5f1c2227 100644 --- a/cmd/help.go +++ b/cmd/help.go @@ -6,6 +6,7 @@ import ( "log" "os" "regexp" + "sort" "strings" "github.com/rclone/rclone/fs" @@ -362,4 +363,28 @@ func showBackend(name string) { fmt.Printf("\n") } } + if backend.MetadataInfo != nil { + fmt.Printf("### Metadata\n\n") + fmt.Printf("%s\n\n", strings.TrimSpace(backend.MetadataInfo.Help)) + if len(backend.MetadataInfo.System) > 0 { + fmt.Printf("Here are the possible system metadata items for the %s backend.\n\n", backend.Name) + keys := []string{} + for k := range backend.MetadataInfo.System { + keys = append(keys, k) + } + sort.Strings(keys) + fmt.Printf("| Name | Help | Type | Example | Read Only |\n") + fmt.Printf("|------|------|------|---------|-----------|\n") + for _, k := range keys { + v := backend.MetadataInfo.System[k] + ro := "N" + if v.ReadOnly { + ro = "**Y**" + } + fmt.Printf("| %s | %s | %s | %s | %s |\n", k, v.Help, v.Type, v.Example, ro) + } + fmt.Printf("\n") + } + fmt.Printf("See the [metadata](/docs/#metadata) docs for more info.\n\n") + } } diff --git a/fs/metadata.go b/fs/metadata.go index dcdb517dc..3df5a29f5 100644 --- a/fs/metadata.go +++ b/fs/metadata.go @@ -7,6 +7,20 @@ import "context" // See docs/content/metadata.md for the interpretation of the keys type Metadata map[string]string +// MetadataHelp represents help for a bit of system metadata +type MetadataHelp struct { + Help string + Type string + Example string + ReadOnly bool +} + +// MetadataInfo is help for the whole metadata for this backend. +type MetadataInfo struct { + System map[string]MetadataHelp + Help string +} + // Set k to v on m // // If m is nil, then it will get made diff --git a/fs/operations/operations.go b/fs/operations/operations.go index a9c2b524c..aa69a0cbf 100644 --- a/fs/operations/operations.go +++ b/fs/operations/operations.go @@ -2278,21 +2278,30 @@ type FsInfo struct { // Features returns the optional features of this Fs Features map[string]bool + + // MetadataInfo returns info about the metadata for this backend + MetadataInfo *fs.MetadataInfo } // GetFsInfo gets the information (FsInfo) about a given Fs func GetFsInfo(f fs.Fs) *FsInfo { + features := f.Features() info := &FsInfo{ - Name: f.Name(), - Root: f.Root(), - String: f.String(), - Precision: f.Precision(), - Hashes: make([]string, 0, 4), - Features: f.Features().Enabled(), + Name: f.Name(), + Root: f.Root(), + String: f.String(), + Precision: f.Precision(), + Hashes: make([]string, 0, 4), + Features: features.Enabled(), + MetadataInfo: nil, } for _, hashType := range f.Hashes().Array() { info.Hashes = append(info.Hashes, hashType.String()) } + fsInfo, _, _, _, err := fs.ParseRemote(fs.ConfigString(f)) + if err == nil && fsInfo != nil && fsInfo.MetadataInfo != nil { + info.MetadataInfo = fsInfo.MetadataInfo + } return info } diff --git a/fs/operations/rc.go b/fs/operations/rc.go index b8c59ce2d..bfa7e2599 100644 --- a/fs/operations/rc.go +++ b/fs/operations/rc.go @@ -413,46 +413,103 @@ This returns info about the remote passed in; ` + "```" + ` { - // optional features and whether they are available or not - "Features": { - "About": true, - "BucketBased": false, - "CanHaveEmptyDirectories": true, - "CaseInsensitive": false, - "ChangeNotify": false, - "CleanUp": false, - "Copy": false, - "DirCacheFlush": false, - "DirMove": true, - "DuplicateFiles": false, - "GetTier": false, - "ListR": false, - "MergeDirs": false, - "Move": true, - "OpenWriterAt": true, - "PublicLink": false, - "Purge": true, - "PutStream": true, - "PutUnchecked": false, - "ReadMimeType": false, - "ServerSideAcrossConfigs": false, - "SetTier": false, - "SetWrapper": false, - "UnWrap": false, - "WrapFs": false, - "WriteMimeType": false - }, - // Names of hashes available - "Hashes": [ - "MD5", - "SHA-1", - "DropboxHash", - "QuickXorHash" - ], - "Name": "local", // Name as created - "Precision": 1, // Precision of timestamps in ns - "Root": "/", // Path as created - "String": "Local file system at /" // how the remote will appear in logs + // optional features and whether they are available or not + "Features": { + "About": true, + "BucketBased": false, + "BucketBasedRootOK": false, + "CanHaveEmptyDirectories": true, + "CaseInsensitive": false, + "ChangeNotify": false, + "CleanUp": false, + "Command": true, + "Copy": false, + "DirCacheFlush": false, + "DirMove": true, + "Disconnect": false, + "DuplicateFiles": false, + "GetTier": false, + "IsLocal": true, + "ListR": false, + "MergeDirs": false, + "MetadataInfo": true, + "Move": true, + "OpenWriterAt": true, + "PublicLink": false, + "Purge": true, + "PutStream": true, + "PutUnchecked": false, + "ReadMetadata": true, + "ReadMimeType": false, + "ServerSideAcrossConfigs": false, + "SetTier": false, + "SetWrapper": false, + "Shutdown": false, + "SlowHash": true, + "SlowModTime": false, + "UnWrap": false, + "UserInfo": false, + "UserMetadata": true, + "WrapFs": false, + "WriteMetadata": true, + "WriteMimeType": false + }, + // Names of hashes available + "Hashes": [ + "md5", + "sha1", + "whirlpool", + "crc32", + "sha256", + "dropbox", + "mailru", + "quickxor" + ], + "Name": "local", // Name as created + "Precision": 1, // Precision of timestamps in ns + "Root": "/", // Path as created + "String": "Local file system at /", // how the remote will appear in logs + // Information about the system metadata for this backend + "MetadataInfo": { + "System": { + "atime": { + "Help": "Time of last access", + "Type": "RFC 3339", + "Example": "2006-01-02T15:04:05.999999999Z07:00" + }, + "btime": { + "Help": "Time of file birth (creation)", + "Type": "RFC 3339", + "Example": "2006-01-02T15:04:05.999999999Z07:00" + }, + "gid": { + "Help": "Group ID of owner", + "Type": "decimal number", + "Example": "500" + }, + "mode": { + "Help": "File type and mode", + "Type": "octal, unix style", + "Example": "0100664" + }, + "mtime": { + "Help": "Time of last modification", + "Type": "RFC 3339", + "Example": "2006-01-02T15:04:05.999999999Z07:00" + }, + "rdev": { + "Help": "Device ID (if special file)", + "Type": "hexadecimal", + "Example": "1abc" + }, + "uid": { + "Help": "User ID of owner", + "Type": "decimal number", + "Example": "500" + } + }, + "Help": "Textual help string\n" + } } ` + "```" + ` diff --git a/fs/registry.go b/fs/registry.go index 2f4afc7fe..d380c7e0c 100644 --- a/fs/registry.go +++ b/fs/registry.go @@ -40,6 +40,8 @@ type RegInfo struct { Aliases []string // Hide - if set don't show in the configurator Hide bool + // MetadataInfo help about the metadata in use in this backend + MetadataInfo *MetadataInfo } // FileName returns the on disk file name for this backend diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go index 47b841bca..5d24296a0 100644 --- a/fstest/fstests/fstests.go +++ b/fstest/fstests/fstests.go @@ -1384,8 +1384,13 @@ func Run(t *testing.T, opt *Opt) { skipIfNotOk(t) features := f.Features() obj := findObject(ctx, t, f, file1.Path) - do, ok := obj.(fs.Metadataer) - if !ok { + do, objectHasMetadata := obj.(fs.Metadataer) + if objectHasMetadata || features.ReadMetadata || features.WriteMetadata || features.UserMetadata { + fsInfo, _, _, _, err := fs.ParseRemote(fs.ConfigString(f)) + require.NoError(t, err) + require.NotNil(t, fsInfo.MetadataInfo, "Object declares metadata support but no MetadataInfo in RegInfo") + } + if !objectHasMetadata { require.False(t, features.ReadMetadata, "Features.ReadMetadata is set but Object.Metadata method not found") t.Skip("Metadata method not supported") }