Add initial S3 support
* Requires a patched goamz to enable metadata reading and writing * Still todo: updating metadata * EU region not working yet
This commit is contained in:
Nick Craig-Wood
parent
f7197c30d7
commit
b9e468ebbd
3
fs.go
3
fs.go
|
|
@ -46,6 +46,9 @@ func NewFs(path string) (Fs, error) {
|
|||
if swiftMatch.MatchString(path) {
|
||||
return NewFsSwift(path)
|
||||
}
|
||||
if s3Match.MatchString(path) {
|
||||
return NewFsS3(path)
|
||||
}
|
||||
return NewFsLocal(path)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,366 @@
|
|||
// S3 interface
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"github.com/ncw/swift"
|
||||
"io"
|
||||
"launchpad.net/goamz/aws"
|
||||
"launchpad.net/goamz/s3"
|
||||
"log"
|
||||
"mime"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Constants
|
||||
const (
|
||||
metaMtime = "X-Amz-Meta-Mtime" // the meta key to store mtime in
|
||||
)
|
||||
|
||||
// FsS3 represents a remote s3 server
|
||||
type FsS3 struct {
|
||||
c *s3.S3 // the connection to the s3 server
|
||||
b *s3.Bucket // the connection to the bucket
|
||||
bucket string // the bucket we are working on
|
||||
perm s3.ACL // permissions for new buckets / objects
|
||||
}
|
||||
|
||||
// FsObjectS3 describes a s3 object
|
||||
type FsObjectS3 struct {
|
||||
// Will definitely have everything but meta which may be nil
|
||||
//
|
||||
// List will read everything but meta - to fill that in need to call
|
||||
// readMetaData
|
||||
s3 *FsS3 // what this object is part of
|
||||
remote string // The remote path
|
||||
etag string // md5sum of the object
|
||||
bytes int64 // size of the object
|
||||
lastModified time.Time // Last modified
|
||||
meta s3.Headers // The object metadata if known - may be nil
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------
|
||||
|
||||
// Globals
|
||||
var (
|
||||
// Flags
|
||||
awsAccessKeyId = flag.String("aws-access-key-id", os.Getenv("AWS_ACCESS_KEY_ID"), "AWS Access Key ID. Defaults to environment var AWS_ACCESS_KEY_ID.")
|
||||
awsSecretAccessKey = flag.String("aws-secret-access-key", os.Getenv("AWS_SECRET_ACCESS_KEY"), "AWS Secret Access Key (password). Defaults to environment var AWS_SECRET_ACCESS_KEY.")
|
||||
// AWS endpoints: http://docs.amazonwebservices.com/general/latest/gr/rande.html#s3_region
|
||||
s3Endpoint = flag.String("s3-endpoint", os.Getenv("S3_ENDPOINT"), "S3 Endpoint. Defaults to environment var S3_ENDPOINT then https://s3.amazonaws.com/.")
|
||||
s3LocationConstraint = flag.String("s3-location-constraint", os.Getenv("S3_LOCATION_CONSTRAINT"), "Location constraint for creating buckets only. Defaults to environment var S3_LOCATION_CONSTRAINT.")
|
||||
)
|
||||
|
||||
// String converts this FsS3 to a string
|
||||
func (f *FsS3) String() string {
|
||||
return fmt.Sprintf("S3 bucket %s", f.bucket)
|
||||
}
|
||||
|
||||
// Pattern to match a s3 url
|
||||
var s3Match = regexp.MustCompile(`^s3://([^/]+)(.*)$`)
|
||||
|
||||
// parseParse parses a s3 'url'
|
||||
func s3ParsePath(path string) (bucket, directory string, err error) {
|
||||
parts := s3Match.FindAllStringSubmatch(path, -1)
|
||||
if len(parts) != 1 || len(parts[0]) != 3 {
|
||||
err = fmt.Errorf("Couldn't parse s3 url %q", path)
|
||||
} else {
|
||||
bucket, directory = parts[0][1], parts[0][2]
|
||||
directory = strings.Trim(directory, "/")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// s3Connection makes a connection to s3
|
||||
func s3Connection() (*s3.S3, error) {
|
||||
// Make the auth
|
||||
if *awsAccessKeyId == "" {
|
||||
return nil, errors.New("Need -aws-access-key-id or environmental variable AWS_ACCESS_KEY_ID")
|
||||
}
|
||||
if *awsSecretAccessKey == "" {
|
||||
return nil, errors.New("Need -aws-secret-access-key or environmental variable AWS_SECRET_ACCESS_KEY")
|
||||
}
|
||||
auth := aws.Auth{*awsAccessKeyId, *awsSecretAccessKey}
|
||||
|
||||
// FIXME look through all the regions by name and use one of them if found
|
||||
|
||||
// Synthesize the region
|
||||
if *s3Endpoint == "" {
|
||||
*s3Endpoint = "https://s3.amazonaws.com/"
|
||||
}
|
||||
region := aws.Region{
|
||||
Name: "s3",
|
||||
S3Endpoint: *s3Endpoint,
|
||||
S3LocationConstraint: false,
|
||||
}
|
||||
if *s3LocationConstraint != "" {
|
||||
region.Name = *s3LocationConstraint
|
||||
region.S3LocationConstraint = true
|
||||
}
|
||||
|
||||
c := s3.New(auth, region)
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// NewFsS3 contstructs an FsS3 from the path, bucket:path
|
||||
func NewFsS3(path string) (*FsS3, error) {
|
||||
bucket, directory, err := s3ParsePath(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if directory != "" {
|
||||
return nil, fmt.Errorf("Directories not supported yet in %q: %q", path, directory)
|
||||
}
|
||||
c, err := s3Connection()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f := &FsS3{
|
||||
c: c,
|
||||
bucket: bucket,
|
||||
b: c.Bucket(bucket),
|
||||
perm: s3.Private, // FIXME need user to specify
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// Lists the buckets
|
||||
func S3Buckets() {
|
||||
c, err := s3Connection()
|
||||
if err != nil {
|
||||
stats.Error()
|
||||
log.Fatalf("Couldn't connect: %s", err)
|
||||
}
|
||||
buckets, err := c.List()
|
||||
if err != nil {
|
||||
stats.Error()
|
||||
log.Fatalf("Couldn't list buckets: %s", err)
|
||||
}
|
||||
for _, bucket := range buckets.Buckets {
|
||||
fmt.Printf("%12s %s\n", bucket.CreationDate, bucket.Name)
|
||||
}
|
||||
}
|
||||
|
||||
// Return an FsObject from a path
|
||||
//
|
||||
// May return nil if an error occurred
|
||||
func (f *FsS3) NewFsObjectWithInfo(remote string, info *s3.Key) FsObject {
|
||||
fs := &FsObjectS3{
|
||||
s3: f,
|
||||
remote: remote,
|
||||
}
|
||||
if info != nil {
|
||||
// Set info but not meta
|
||||
var err error
|
||||
fs.lastModified, err = time.Parse(time.RFC3339, info.LastModified)
|
||||
if err != nil {
|
||||
FsLog(fs, "Failed to read last modified: %s", err)
|
||||
fs.lastModified = time.Now()
|
||||
}
|
||||
fs.etag = info.ETag
|
||||
fs.bytes = info.Size
|
||||
} else {
|
||||
err := fs.readMetaData() // reads info and meta, returning an error
|
||||
if err != nil {
|
||||
// logged already FsDebug("Failed to read info: %s", err)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return fs
|
||||
}
|
||||
|
||||
// Return an FsObject from a path
|
||||
//
|
||||
// May return nil if an error occurred
|
||||
func (f *FsS3) NewFsObject(remote string) FsObject {
|
||||
return f.NewFsObjectWithInfo(remote, nil)
|
||||
}
|
||||
|
||||
// Walk the path returning a channel of FsObjects
|
||||
func (f *FsS3) List() FsObjectsChan {
|
||||
out := make(FsObjectsChan, *checkers)
|
||||
go func() {
|
||||
// FIXME need to implement ALL loop
|
||||
objects, err := f.b.List("", "", "", 10000)
|
||||
if err != nil {
|
||||
stats.Error()
|
||||
log.Printf("Couldn't read bucket %q: %s", f.bucket, err)
|
||||
} else {
|
||||
for i := range objects.Contents {
|
||||
object := &objects.Contents[i]
|
||||
if fs := f.NewFsObjectWithInfo(object.Key, object); fs != nil {
|
||||
out <- fs
|
||||
}
|
||||
}
|
||||
}
|
||||
close(out)
|
||||
}()
|
||||
return out
|
||||
}
|
||||
|
||||
// Put the FsObject into the bucket
|
||||
func (f *FsS3) Put(src FsObject) {
|
||||
// Temporary FsObject under construction
|
||||
fs := &FsObjectS3{s3: f, remote: src.Remote()}
|
||||
|
||||
in0, err := src.Open()
|
||||
if err != nil {
|
||||
stats.Error()
|
||||
FsLog(fs, "Failed to open: %s", err)
|
||||
return
|
||||
}
|
||||
in := NewAccount(in0) // account the transfer
|
||||
defer in.Close()
|
||||
|
||||
// Set the mtime in the headers
|
||||
headers := s3.Headers{
|
||||
metaMtime: swift.TimeToFloatString(src.ModTime()),
|
||||
}
|
||||
|
||||
// Guess the content type
|
||||
contentType := mime.TypeByExtension(path.Ext(fs.remote))
|
||||
if contentType == "" {
|
||||
contentType = "application/octet-stream"
|
||||
}
|
||||
|
||||
_, err = fs.s3.b.PutReaderHeaders(fs.remote, in, src.Size(), contentType, fs.s3.perm, headers)
|
||||
if err != nil {
|
||||
stats.Error()
|
||||
FsLog(fs, "Failed to upload: %s", err)
|
||||
FsDebug(fs, "Removing failed upload")
|
||||
removeErr := fs.Remove()
|
||||
if removeErr != nil {
|
||||
stats.Error()
|
||||
FsLog(fs, "Failed to remove failed download: %s", removeErr)
|
||||
}
|
||||
return
|
||||
}
|
||||
FsDebug(fs, "Uploaded")
|
||||
}
|
||||
|
||||
// Mkdir creates the bucket if it doesn't exist
|
||||
func (f *FsS3) Mkdir() error {
|
||||
return f.b.PutBucket(f.perm)
|
||||
}
|
||||
|
||||
// Rmdir deletes the bucket
|
||||
//
|
||||
// Returns an error if it isn't empty
|
||||
func (f *FsS3) Rmdir() error {
|
||||
return f.b.DelBucket()
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------
|
||||
|
||||
// Return the remote path
|
||||
func (fs *FsObjectS3) Remote() string {
|
||||
return fs.remote
|
||||
}
|
||||
|
||||
// Md5sum returns the Md5sum of an object returning a lowercase hex string
|
||||
func (fs *FsObjectS3) Md5sum() (string, error) {
|
||||
return strings.Trim(strings.ToLower(fs.etag), `"`), nil
|
||||
}
|
||||
|
||||
// Size returns the size of an object in bytes
|
||||
func (fs *FsObjectS3) Size() int64 {
|
||||
return fs.bytes
|
||||
}
|
||||
|
||||
// readMetaData gets the metadata if it hasn't already been fetched
|
||||
//
|
||||
// it also sets the info
|
||||
func (fs *FsObjectS3) readMetaData() (err error) {
|
||||
if fs.meta != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
headers, err := fs.s3.b.Head(fs.remote, nil)
|
||||
if err != nil {
|
||||
FsDebug(fs, "Failed to read info: %s", err)
|
||||
return err
|
||||
}
|
||||
size, err := strconv.ParseInt(headers["Content-Length"], 10, 64)
|
||||
if err != nil {
|
||||
FsDebug(fs, "Failed to read size from: %q", headers)
|
||||
return err
|
||||
}
|
||||
fs.etag = headers["Etag"]
|
||||
fs.bytes = size
|
||||
fs.meta = headers
|
||||
if fs.lastModified, err = time.Parse(http.TimeFormat, headers["Last-Modified"]); err != nil {
|
||||
FsLog(fs, "Failed to read last modified from HEAD: %s", err)
|
||||
fs.lastModified = time.Now()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ModTime returns the modification time of the object
|
||||
//
|
||||
// It attempts to read the objects mtime and if that isn't present the
|
||||
// LastModified returned in the http headers
|
||||
func (fs *FsObjectS3) ModTime() time.Time {
|
||||
err := fs.readMetaData()
|
||||
if err != nil {
|
||||
FsLog(fs, "Failed to read metadata: %s", err)
|
||||
return time.Now()
|
||||
}
|
||||
// read mtime out of metadata if available
|
||||
d, ok := fs.meta[metaMtime]
|
||||
if !ok {
|
||||
// FsDebug(fs, "No metadata")
|
||||
return fs.lastModified
|
||||
}
|
||||
modTime, err := swift.FloatStringToTime(d)
|
||||
if err != nil {
|
||||
FsLog(fs, "Failed to read mtime from object: %s", err)
|
||||
return fs.lastModified
|
||||
}
|
||||
return modTime
|
||||
}
|
||||
|
||||
// Sets the modification time of the local fs object
|
||||
func (fs *FsObjectS3) SetModTime(modTime time.Time) {
|
||||
// FIXME goamz can't set metadata, though it is possible with a PUT and copy
|
||||
FsLog(fs, "Can't set mod time yet: %s", modTime)
|
||||
// err := fs.readMetaData()
|
||||
// if err != nil {
|
||||
// stats.Error()
|
||||
// FsLog(fs, "Failed to read metadata: %s", err)
|
||||
// return
|
||||
// }
|
||||
// fs.meta.SetModTime(modTime)
|
||||
// err = fs.s3.c.ObjectUpdate(fs.s3.bucket, fs.remote, fs.meta.ObjectHeaders())
|
||||
// if err != nil {
|
||||
// stats.Error()
|
||||
// FsLog(fs, "Failed to update remote mtime: %s", err)
|
||||
// }
|
||||
}
|
||||
|
||||
// Is this object storable
|
||||
func (fs *FsObjectS3) Storable() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// Open an object for read
|
||||
func (fs *FsObjectS3) Open() (in io.ReadCloser, err error) {
|
||||
in, err = fs.s3.b.GetReader(fs.remote)
|
||||
return
|
||||
}
|
||||
|
||||
// Remove an object
|
||||
func (fs *FsObjectS3) Remove() error {
|
||||
return fs.s3.b.Del(fs.remote)
|
||||
}
|
||||
|
||||
// Check the interfaces are satisfied
|
||||
var _ Fs = &FsS3{}
|
||||
var _ FsObject = &FsObjectS3{}
|
||||
|
|
@ -47,7 +47,7 @@ func (f *FsSwift) String() string {
|
|||
}
|
||||
|
||||
// Pattern to match a swift url
|
||||
var swiftMatch = regexp.MustCompile(`^([^/:]+):(.*)$`)
|
||||
var swiftMatch = regexp.MustCompile(`^swift://([^/]+)(.*)$`)
|
||||
|
||||
// parseParse parses a swift 'url'
|
||||
func parsePath(path string) (container, directory string, err error) {
|
||||
|
|
@ -294,7 +294,7 @@ func (fs *FsObjectSwift) Storable() bool {
|
|||
|
||||
// Open an object for read
|
||||
func (fs *FsObjectSwift) Open() (in io.ReadCloser, err error) {
|
||||
in, _, err = fs.swift.c.ObjectOpen(fs.swift.container, fs.info.Name, true, nil)
|
||||
in, _, err = fs.swift.c.ObjectOpen(fs.swift.container, fs.remote, true, nil)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
Todo
|
||||
* Factor fses into own packages
|
||||
* FIXME: ls without an argument for buckets/containers?
|
||||
* FIXME: More -dry-run checks for object transfer
|
||||
* FIXME: EU Region not working
|
||||
* Ignoring the pseudo directories
|
||||
* if object.PseudoDirectory {
|
||||
* fmt.Printf("%9s %19s %s\n", "Directory", "-", fs.Remote())
|
||||
|
|
@ -14,8 +17,10 @@ Todo
|
|||
* Windows paths? Do we need to translate / and \?
|
||||
* Make a fs.Errorf and count errors and log them at a different level
|
||||
* add -modify-window flag - fs should keep knowledge of resolution
|
||||
* Add max object size to fs metadata - 5GB for swift, infinite for local, ? for s3
|
||||
|
||||
Ideas
|
||||
* could do encryption - put IV into metadata?
|
||||
* optimise remote copy container to another container using remote
|
||||
copy if local is same as remote
|
||||
* Allow subpaths container:/sub/path
|
||||
|
|
|
|||
|
|
@ -283,7 +283,8 @@ func List(f Fs) {
|
|||
// Lists files in a container
|
||||
func list(fdst, fsrc Fs) {
|
||||
if fdst == nil {
|
||||
SwiftContainers()
|
||||
// FIXMESwiftContainers()
|
||||
S3Buckets()
|
||||
return
|
||||
}
|
||||
List(fdst)
|
||||
|
|
|
|||
Loading…
Reference in New Issue