Detect Fs from path to make operations consistent everywhere
This commit is contained in:
Nick Craig-Wood
parent
4028a4192f
commit
c15ae179ee
10
fs.go
10
fs.go
|
|
@ -37,6 +37,16 @@ type FsObjectsChan chan FsObject
|
|||
|
||||
type FsObjects []FsObject
|
||||
|
||||
// NewFs makes a new Fs object from the path
|
||||
//
|
||||
// FIXME make more generic in future
|
||||
func NewFs(path string) (Fs, error) {
|
||||
if swiftMatch.MatchString(path) {
|
||||
return NewFsSwift(path)
|
||||
}
|
||||
return NewFsLocal(path)
|
||||
}
|
||||
|
||||
// checkClose is a utility function used to check the return from
|
||||
// Close in a defer statement.
|
||||
func checkClose(c io.Closer, err *error) {
|
||||
|
|
|
|||
|
|
@ -26,6 +26,13 @@ type FsObjectLocal struct {
|
|||
|
||||
// ------------------------------------------------------------
|
||||
|
||||
// NewFsLocal contstructs an FsLocal from the path
|
||||
func NewFsLocal(root string) (*FsLocal, error) {
|
||||
root = path.Clean(root)
|
||||
f := &FsLocal{root: root}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// Return an FsObject from a path
|
||||
//
|
||||
// May return nil if an error occurred
|
||||
|
|
|
|||
83
fs_swift.go
83
fs_swift.go
|
|
@ -2,10 +2,14 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"github.com/ncw/swift"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
|
@ -28,6 +32,85 @@ type FsObjectSwift struct {
|
|||
|
||||
// ------------------------------------------------------------
|
||||
|
||||
// Globals
|
||||
var (
|
||||
// Flags
|
||||
// FIXME make these part of swift so we get a standard set of flags?
|
||||
authUrl = flag.String("auth", os.Getenv("ST_AUTH"), "Auth URL for server. Defaults to environment var ST_AUTH.")
|
||||
userName = flag.String("user", os.Getenv("ST_USER"), "User name. Defaults to environment var ST_USER.")
|
||||
apiKey = flag.String("key", os.Getenv("ST_KEY"), "API key (password). Defaults to environment var ST_KEY.")
|
||||
)
|
||||
|
||||
// Pattern to match a swift url
|
||||
var swiftMatch = regexp.MustCompile(`^([^/:]+):(.*)$`)
|
||||
|
||||
// parseParse parses a swift 'url'
|
||||
func parsePath(path string) (container, directory string, err error) {
|
||||
parts := swiftMatch.FindAllStringSubmatch(path, -1)
|
||||
if len(parts) != 1 || len(parts[0]) != 3 {
|
||||
err = fmt.Errorf("Couldn't parse swift url %q", path)
|
||||
} else {
|
||||
container, directory = parts[0][1], parts[0][2]
|
||||
directory = strings.Trim(directory, "/")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// swiftConnection makes a connection to swift
|
||||
func swiftConnection() (*swift.Connection, error) {
|
||||
if *userName == "" {
|
||||
return nil, errors.New("Need -user or environmental variable ST_USER")
|
||||
}
|
||||
if *apiKey == "" {
|
||||
return nil, errors.New("Need -key or environmental variable ST_KEY")
|
||||
}
|
||||
if *authUrl == "" {
|
||||
return nil, errors.New("Need -auth or environmental variable ST_AUTH")
|
||||
}
|
||||
c := &swift.Connection{
|
||||
UserName: *userName,
|
||||
ApiKey: *apiKey,
|
||||
AuthUrl: *authUrl,
|
||||
}
|
||||
err := c.Authenticate()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// NewFsSwift contstructs an FsSwift from the path, container:path
|
||||
func NewFsSwift(path string) (*FsSwift, error) {
|
||||
container, directory, err := parsePath(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if directory != "" {
|
||||
return nil, fmt.Errorf("Directories not supported yet in %q", path)
|
||||
}
|
||||
c, err := swiftConnection()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f := &FsSwift{c: *c, container: container}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// Lists the containers
|
||||
func SwiftContainers() {
|
||||
c, err := swiftConnection()
|
||||
if err != nil {
|
||||
log.Fatalf("Couldn't connect: %s", err)
|
||||
}
|
||||
containers, err := c.ContainersAll(nil)
|
||||
if err != nil {
|
||||
log.Fatalf("Couldn't list containers: %s", err)
|
||||
}
|
||||
for _, container := range containers {
|
||||
fmt.Printf("%9d %12d %s\n", container.Count, container.Bytes, container.Name)
|
||||
}
|
||||
}
|
||||
|
||||
// Return an FsObject from a path
|
||||
//
|
||||
// May return nil if an error occurred
|
||||
|
|
|
|||
65
notes.txt
65
notes.txt
|
|
@ -1,40 +1,24 @@
|
|||
Todo
|
||||
* Add sync command (like rsync with delete)
|
||||
* Check logging in various parts
|
||||
* Make logging controllable with flags
|
||||
* progress meter would be nice! Do this by wrapping the Reader with a progress bar
|
||||
* Do bandwidth limit by wrapping the Reader too
|
||||
* Maybe using https://jra-go.googlecode.com/hg/linkio/ which will work for multiple
|
||||
uploads or downloads.
|
||||
* code.google.com/p/mxk/go1/flowcontrol - only does one flow at once
|
||||
* Or maybe put into swift library.
|
||||
* Make swift timeouts be settable with command line parameters
|
||||
* Check the locking in swift module!
|
||||
* Windows paths? Do we need to translate / and \?
|
||||
* Make a fs.Errorf and count errors and log them at a different level
|
||||
|
||||
Ideas
|
||||
* remote copy container to another container
|
||||
* check local is same as remote
|
||||
* syncup/syncdown (like rsync with delete)
|
||||
* use container: /path/syntax like rsync?
|
||||
* optimise remote copy container to another container using remote
|
||||
copy if local is same as remote
|
||||
* Allow subpaths container:/sub/path
|
||||
* allow local/local too
|
||||
* progress reports
|
||||
* stats
|
||||
* Add bandwidth limit?
|
||||
|
||||
make 100% compatible with swift.py?
|
||||
|
||||
Make Env vars compatible with st?
|
||||
|
||||
Get and put the metadata in the libray (x-object-meta-mtime) when getting and putting a file?
|
||||
|
||||
This also puts meta-mtime
|
||||
https://github.com/gholt/swiftly
|
||||
|
||||
As an integer, but it does parse it as a float
|
||||
subargs.append('x-object-meta-mtime:%d' % getmtime(options.input_))
|
||||
|
||||
Need an iterate all objects routine... Could use a channel
|
||||
- could just be a flag to Objects()
|
||||
|
||||
FIXME progress meter would be nice! Do this by wrapping the Reader with a progress bar
|
||||
|
||||
Do bandwidth limit by wrapping the Reader too
|
||||
Maybe using https://jra-go.googlecode.com/hg/linkio/ which will work for multiple
|
||||
uploads or downloads.
|
||||
* code.google.com/p/mxk/go1/flowcontrol - only does one flow at once
|
||||
Or maybe put into swift library.
|
||||
|
||||
Windows paths? Do we need to translate / and \?
|
||||
|
||||
Make swift timeouts be settable with command line parameters
|
||||
* look at auth from env in s3 module - add to swift?
|
||||
|
||||
Make a wrapper in connection which
|
||||
* measures bandwidth and reports it
|
||||
|
|
@ -43,10 +27,17 @@ Make a wrapper in connection which
|
|||
* does timeouts by setting a limit, seeing whether io has happened
|
||||
and resetting it if it has
|
||||
|
||||
Check the locking in swift module!
|
||||
|
||||
Need to make directory objects otherwise can't upload an empty directory
|
||||
* Or could upload empty directories only?
|
||||
* Can't purge a local filesystem because it leaves the directories behind
|
||||
|
||||
Make a fs.Errorf and count errors and log them at a different level
|
||||
s3
|
||||
--
|
||||
|
||||
Can maybe set last modified?
|
||||
|
||||
https://forums.aws.amazon.com/message.jspa?messageID=214062
|
||||
|
||||
Otherwise can set metadata
|
||||
|
||||
Returns etag and last modified in bucket list
|
||||
|
|
|
|||
202
swiftsync.go
202
swiftsync.go
|
|
@ -6,7 +6,6 @@ package main
|
|||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"github.com/ncw/swift"
|
||||
"log"
|
||||
"os"
|
||||
"runtime"
|
||||
|
|
@ -22,12 +21,8 @@ var (
|
|||
snet = flag.Bool("snet", false, "Use internal service network") // FIXME not implemented
|
||||
verbose = flag.Bool("verbose", false, "Print lots more stuff")
|
||||
quiet = flag.Bool("quiet", false, "Print as little stuff as possible")
|
||||
// FIXME make these part of swift so we get a standard set of flags?
|
||||
authUrl = flag.String("auth", os.Getenv("ST_AUTH"), "Auth URL for server. Defaults to environment var ST_AUTH.")
|
||||
userName = flag.String("user", os.Getenv("ST_USER"), "User name. Defaults to environment var ST_USER.")
|
||||
apiKey = flag.String("key", os.Getenv("ST_KEY"), "API key (password). Defaults to environment var ST_KEY.")
|
||||
checkers = flag.Int("checkers", 8, "Number of checkers to run in parallel.")
|
||||
transfers = flag.Int("transfers", 4, "Number of file transfers to run in parallel.")
|
||||
checkers = flag.Int("checkers", 8, "Number of checkers to run in parallel.")
|
||||
transfers = flag.Int("transfers", 4, "Number of file transfers to run in parallel.")
|
||||
)
|
||||
|
||||
// Read FsObjects~s on in send to out if they need uploading
|
||||
|
|
@ -65,7 +60,7 @@ func Copier(in FsObjectsChan, fdst Fs, wg *sync.WaitGroup) {
|
|||
}
|
||||
|
||||
// Copies fsrc into fdst
|
||||
func Copy(fsrc, fdst Fs) {
|
||||
func Copy(fdst, fsrc Fs) {
|
||||
err := fdst.Mkdir()
|
||||
if err != nil {
|
||||
log.Fatal("Failed to make destination")
|
||||
|
|
@ -93,40 +88,9 @@ func Copy(fsrc, fdst Fs) {
|
|||
copierWg.Wait()
|
||||
}
|
||||
|
||||
// Syncs a directory into a container
|
||||
func upload(c *swift.Connection, args []string) {
|
||||
root, container := args[0], args[1]
|
||||
// FIXME
|
||||
fsrc := &FsLocal{root: root}
|
||||
fdst := &FsSwift{c: *c, container: container}
|
||||
|
||||
Copy(fsrc, fdst)
|
||||
}
|
||||
|
||||
// Syncs a container into a directory
|
||||
//
|
||||
// FIXME need optional stat in FsObject and to be able to make FsObjects from ObjectsAll
|
||||
//
|
||||
// FIXME should download and stat many at once
|
||||
func download(c *swift.Connection, args []string) {
|
||||
container, root := args[0], args[1]
|
||||
|
||||
// FIXME
|
||||
fsrc := &FsSwift{c: *c, container: container}
|
||||
fdst := &FsLocal{root: root}
|
||||
|
||||
Copy(fsrc, fdst)
|
||||
}
|
||||
|
||||
// Lists the containers
|
||||
func listContainers(c *swift.Connection) {
|
||||
containers, err := c.ContainersAll(nil)
|
||||
if err != nil {
|
||||
log.Fatalf("Couldn't list containers: %s", err)
|
||||
}
|
||||
for _, container := range containers {
|
||||
fmt.Printf("%9d %12d %s\n", container.Count, container.Bytes, container.Name)
|
||||
}
|
||||
// Copy~s from source to dest
|
||||
func copy_(fdst, fsrc Fs) {
|
||||
Copy(fdst, fsrc)
|
||||
}
|
||||
|
||||
// List the Fs to stdout
|
||||
|
|
@ -147,57 +111,34 @@ func List(f Fs) {
|
|||
}
|
||||
|
||||
// Lists files in a container
|
||||
func list(c *swift.Connection, args []string) {
|
||||
if len(args) == 0 {
|
||||
listContainers(c)
|
||||
func list(fdst, fsrc Fs) {
|
||||
if fdst == nil {
|
||||
SwiftContainers()
|
||||
return
|
||||
}
|
||||
container := args[0]
|
||||
// FIXME
|
||||
f := &FsSwift{c: *c, container: container}
|
||||
List(f)
|
||||
List(fdst)
|
||||
}
|
||||
|
||||
// Local lists files
|
||||
func llist(c *swift.Connection, args []string) {
|
||||
root := args[0]
|
||||
// FIXME
|
||||
f := &FsLocal{root: root}
|
||||
List(f)
|
||||
}
|
||||
|
||||
// Makes a container
|
||||
func mkdir(c *swift.Connection, args []string) {
|
||||
container := args[0]
|
||||
// FIXME
|
||||
fdst := &FsSwift{c: *c, container: container}
|
||||
|
||||
// Makes a destination directory or container
|
||||
func mkdir(fdst, fsrc Fs) {
|
||||
err := fdst.Mkdir()
|
||||
if err != nil {
|
||||
log.Fatalf("Couldn't create container %q: %s", container, err)
|
||||
log.Fatalf("Mkdir failed: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Removes a container
|
||||
func rmdir(c *swift.Connection, args []string) {
|
||||
container := args[0]
|
||||
// FIXME
|
||||
fdst := &FsSwift{c: *c, container: container}
|
||||
|
||||
// Removes a container but not if not empty
|
||||
func rmdir(fdst, fsrc Fs) {
|
||||
err := fdst.Rmdir()
|
||||
if err != nil {
|
||||
log.Fatalf("Couldn't delete container %q: %s", container, err)
|
||||
log.Fatalf("Rmdir failed: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Removes a container and all of its contents
|
||||
//
|
||||
// FIXME should make FsObjects and use debugging
|
||||
func purge(c *swift.Connection, args []string) {
|
||||
container := args[0]
|
||||
// FIXME
|
||||
fdst := &FsSwift{c: *c, container: container}
|
||||
|
||||
// FIXME doesn't delete local directories
|
||||
func purge(fdst, fsrc Fs) {
|
||||
to_be_deleted := fdst.List()
|
||||
|
||||
var wg sync.WaitGroup
|
||||
|
|
@ -219,14 +160,14 @@ func purge(c *swift.Connection, args []string) {
|
|||
log.Printf("Waiting for deletions to finish")
|
||||
wg.Wait()
|
||||
|
||||
log.Printf("Deleting container")
|
||||
rmdir(c, args)
|
||||
log.Printf("Deleting path")
|
||||
rmdir(fdst, fsrc)
|
||||
}
|
||||
|
||||
type Command struct {
|
||||
name string
|
||||
help string
|
||||
run func(*swift.Connection, []string)
|
||||
run func(fdst, fsrc Fs)
|
||||
minArgs, maxArgs int
|
||||
}
|
||||
|
||||
|
|
@ -245,62 +186,55 @@ func (cmd *Command) checkArgs(args []string) {
|
|||
|
||||
var Commands = []Command{
|
||||
{
|
||||
"upload",
|
||||
`<directory> <container>
|
||||
Upload the local directory to the remote container. Doesn't
|
||||
upload unchanged files, testing first by modification time
|
||||
then by MD5SUM
|
||||
"copy",
|
||||
`<source> <destination>
|
||||
|
||||
Copy the source to the destination. Doesn't transfer
|
||||
unchanged files, testing first by modification time then by
|
||||
MD5SUM. Doesn't delete files from the destination.
|
||||
|
||||
`,
|
||||
upload,
|
||||
2, 2,
|
||||
},
|
||||
{
|
||||
"download",
|
||||
`<container> <directory>
|
||||
Download the container to the local directory. Doesn't
|
||||
download unchanged files
|
||||
`,
|
||||
download,
|
||||
copy_,
|
||||
2, 2,
|
||||
},
|
||||
{
|
||||
"ls",
|
||||
`[<container>]
|
||||
List the containers if no parameter supplied or the contents
|
||||
of <container>
|
||||
`[<path>]
|
||||
|
||||
List the path. If no parameter is supplied then it lists the
|
||||
available swift containers.
|
||||
|
||||
`,
|
||||
list,
|
||||
0, 1,
|
||||
},
|
||||
{
|
||||
"lls",
|
||||
`[<directory>]
|
||||
List the directory
|
||||
`,
|
||||
llist,
|
||||
1, 1,
|
||||
},
|
||||
{
|
||||
"mkdir",
|
||||
`<container>
|
||||
Make the container if it doesn't already exist
|
||||
`<path>
|
||||
|
||||
Make the path if it doesn't already exist
|
||||
|
||||
`,
|
||||
mkdir,
|
||||
1, 1,
|
||||
},
|
||||
{
|
||||
"rmdir",
|
||||
`<container>
|
||||
Remove the container. Note that you can't remove a container with
|
||||
objects in - use rm for that
|
||||
`<path>
|
||||
|
||||
Remove the path. Note that you can't remove a path with
|
||||
objects in it, use purge for that
|
||||
|
||||
`,
|
||||
rmdir,
|
||||
1, 1,
|
||||
},
|
||||
{
|
||||
"purge",
|
||||
`<container>
|
||||
Remove the container and all of the contents.
|
||||
`<path>
|
||||
|
||||
Remove the path and all of its contents.
|
||||
|
||||
`,
|
||||
purge,
|
||||
1, 1,
|
||||
|
|
@ -314,6 +248,7 @@ func syntaxError() {
|
|||
Syntax: [options] subcommand <parameters> <parameters...>
|
||||
|
||||
Subcommands:
|
||||
|
||||
`)
|
||||
for i := range Commands {
|
||||
cmd := &Commands[i]
|
||||
|
|
@ -350,30 +285,10 @@ func main() {
|
|||
defer pprof.StopCPUProfile()
|
||||
}
|
||||
|
||||
fmt.Println(args)
|
||||
if len(args) < 1 {
|
||||
fatal("No command supplied\n")
|
||||
}
|
||||
|
||||
if *userName == "" {
|
||||
log.Fatal("Need -user or environmental variable ST_USER")
|
||||
}
|
||||
if *apiKey == "" {
|
||||
log.Fatal("Need -key or environmental variable ST_KEY")
|
||||
}
|
||||
if *authUrl == "" {
|
||||
log.Fatal("Need -auth or environmental variable ST_AUTH")
|
||||
}
|
||||
c := &swift.Connection{
|
||||
UserName: *userName,
|
||||
ApiKey: *apiKey,
|
||||
AuthUrl: *authUrl,
|
||||
}
|
||||
err := c.Authenticate()
|
||||
if err != nil {
|
||||
log.Fatal("Failed to authenticate", err)
|
||||
}
|
||||
|
||||
cmd := strings.ToLower(args[0])
|
||||
args = args[1:]
|
||||
|
||||
|
|
@ -396,5 +311,24 @@ func main() {
|
|||
log.Fatalf("Unknown command %q", cmd)
|
||||
}
|
||||
found.checkArgs(args)
|
||||
found.run(c, args)
|
||||
|
||||
// Make source and destination fs
|
||||
var fdst, fsrc Fs
|
||||
var err error
|
||||
if len(args) >= 1 {
|
||||
fdst, err = NewFs(args[0])
|
||||
if err != nil {
|
||||
log.Fatal("Failed to create file system: ", err)
|
||||
}
|
||||
}
|
||||
if len(args) >= 2 {
|
||||
fsrc, err = NewFs(args[1])
|
||||
if err != nil {
|
||||
log.Fatal("Failed to create destination file system: ", err)
|
||||
}
|
||||
fsrc, fdst = fdst, fsrc
|
||||
}
|
||||
|
||||
// Run the actual command
|
||||
found.run(fdst, fsrc)
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue