rclone/fs/metadata.go

173 lines
4.3 KiB
Go

package fs
import (
"bytes"
"context"
"encoding/json"
"fmt"
"os/exec"
"strings"
"time"
)
// Metadata represents Object metadata in a standardised form
//
// See docs/content/metadata.md for the interpretation of the keys
type Metadata map[string]string
// MetadataHelp represents help for a bit of system metadata
type MetadataHelp struct {
Help string
Type string
Example string
ReadOnly bool
}
// MetadataInfo is help for the whole metadata for this backend.
type MetadataInfo struct {
System map[string]MetadataHelp
Help string
}
// Set k to v on m
//
// If m is nil, then it will get made
func (m *Metadata) Set(k, v string) {
if *m == nil {
*m = make(Metadata, 1)
}
(*m)[k] = v
}
// Merge other into m
//
// If m is nil, then it will get made
func (m *Metadata) Merge(other Metadata) {
for k, v := range other {
if *m == nil {
*m = make(Metadata, len(other))
}
(*m)[k] = v
}
}
// MergeOptions gets any Metadata from the options passed in and
// stores it in m (which may be nil).
//
// If there is no m then metadata will be nil
func (m *Metadata) MergeOptions(options []OpenOption) {
for _, opt := range options {
if metadataOption, ok := opt.(MetadataOption); ok {
m.Merge(Metadata(metadataOption))
}
}
}
// GetMetadata from an ObjectInfo
//
// If the object has no metadata then metadata will be nil
func GetMetadata(ctx context.Context, o ObjectInfo) (metadata Metadata, err error) {
do, ok := o.(Metadataer)
if !ok {
return nil, nil
}
return do.Metadata(ctx)
}
// mapItem descripts the item to be mapped
type mapItem struct {
SrcFs string
SrcFsType string
DstFs string
DstFsType string
Remote string
Size int64
MimeType string `json:",omitempty"`
ModTime time.Time
IsDir bool
ID string `json:",omitempty"`
Metadata Metadata `json:",omitempty"`
}
// This runs an external program on the metadata which can be used to
// map it from one form to another.
func metadataMapper(ctx context.Context, cmdLine SpaceSepList, dstFs Fs, o ObjectInfo, metadata Metadata) (newMetadata Metadata, err error) {
ci := GetConfig(ctx)
cmd := exec.Command(cmdLine[0], cmdLine[1:]...)
in := mapItem{
DstFs: ConfigString(dstFs),
DstFsType: Type(dstFs),
Remote: o.Remote(),
Size: o.Size(),
MimeType: MimeType(ctx, o),
ModTime: o.ModTime(ctx),
IsDir: false,
Metadata: metadata,
}
fInfo := o.Fs()
if f, ok := fInfo.(Fs); ok {
in.SrcFs = ConfigString(f)
in.SrcFsType = Type(f)
} else {
in.SrcFs = fInfo.Name() + ":" + fInfo.Root()
in.SrcFsType = "unknown"
}
if do, ok := o.(IDer); ok {
in.ID = do.ID()
}
inBytes, err := json.MarshalIndent(in, "", "\t")
if err != nil {
return nil, fmt.Errorf("metadata mapper: failed to marshal input: %w", err)
}
if ci.Dump.IsSet(DumpMapper) {
Debugf(nil, "Metadata mapper sent: \n%s\n", string(inBytes))
}
var stdout, stderr bytes.Buffer
cmd.Stdin = bytes.NewBuffer(inBytes)
cmd.Stdout = &stdout
cmd.Stderr = &stderr
start := time.Now()
err = cmd.Run()
Debugf(o, "Calling metadata mapper %v", cmdLine)
duration := time.Since(start)
if err != nil {
return nil, fmt.Errorf("metadata mapper: failed on %v: %q: %w", cmdLine, strings.TrimSpace(stderr.String()), err)
}
if ci.Dump.IsSet(DumpMapper) {
Debugf(nil, "Metadata mapper received: \n%s\n", stdout.String())
}
var out mapItem
err = json.Unmarshal(stdout.Bytes(), &out)
if err != nil {
return nil, fmt.Errorf("metadata mapper: failed to read output: %q: %w", stdout.String(), err)
}
Debugf(o, "Metadata mapper returned in %v", duration)
return out.Metadata, nil
}
// GetMetadataOptions from an ObjectInfo and merge it with any in options
//
// If --metadata isn't in use it will return nil.
//
// If the object has no metadata then metadata will be nil.
//
// This should be passed the destination Fs for the metadata mapper
func GetMetadataOptions(ctx context.Context, dstFs Fs, o ObjectInfo, options []OpenOption) (metadata Metadata, err error) {
ci := GetConfig(ctx)
if !ci.Metadata {
return nil, nil
}
metadata, err = GetMetadata(ctx, o)
if err != nil {
return nil, err
}
metadata.MergeOptions(options)
if len(ci.MetadataMapper) != 0 {
metadata, err = metadataMapper(ctx, ci.MetadataMapper, dstFs, o, metadata)
if err != nil {
return nil, err
}
}
return metadata, nil
}