Remove filesystem flags and put in config file with editor
This commit is contained in:
Nick Craig-Wood
parent
8fd43a52e7
commit
0a108832e2
|
|
@ -3,3 +3,4 @@
|
|||
test-env*
|
||||
_junk/
|
||||
rclone
|
||||
upload
|
||||
|
|
|
|||
76
drive/fs.go
76
drive/fs.go
|
|
@ -22,30 +22,52 @@ package drive
|
|||
// * files with / in name
|
||||
|
||||
import (
|
||||
"code.google.com/p/goauth2/oauth"
|
||||
"code.google.com/p/google-api-go-client/drive/v2"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"github.com/ncw/rclone/fs"
|
||||
"io"
|
||||
"log"
|
||||
"mime"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Pattern to match a drive url
|
||||
var Match = regexp.MustCompile(`^drive://(.*)$`)
|
||||
"code.google.com/p/goauth2/oauth"
|
||||
"code.google.com/p/google-api-go-client/drive/v2"
|
||||
"github.com/ncw/rclone/fs"
|
||||
)
|
||||
|
||||
// Register with Fs
|
||||
func init() {
|
||||
fs.Register(Match, NewFs)
|
||||
fs.Register(&fs.FsInfo{
|
||||
Name: "drive",
|
||||
NewFs: NewFs,
|
||||
Options: []fs.Option{{
|
||||
Name: "client_id",
|
||||
Help: "Google Application Client Id.",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "202264815644.apps.googleusercontent.com",
|
||||
Help: "rclone's client id - use this or your own if you want",
|
||||
}},
|
||||
}, {
|
||||
Name: "client_secret",
|
||||
Help: "Google Application Client Secret.",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "X4Z3ca8xfWDb1Voo-F9a7ZxJ",
|
||||
Help: "rclone's client secret - use this or your own if you want",
|
||||
}},
|
||||
}, {
|
||||
Name: "token_file",
|
||||
Help: "Path to store token file.",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: path.Join(fs.HomeDir, ".gdrive-token-file"),
|
||||
Help: "Suggested path for token file",
|
||||
}},
|
||||
}},
|
||||
})
|
||||
}
|
||||
|
||||
// FsDrive represents a remote drive server
|
||||
|
|
@ -128,11 +150,8 @@ const (
|
|||
// Globals
|
||||
var (
|
||||
// Flags
|
||||
driveClientId = flag.String("drive-client-id", os.Getenv("GDRIVE_CLIENT_ID"), "Auth URL for server. Defaults to environment var GDRIVE_CLIENT_ID.")
|
||||
driveClientSecret = flag.String("drive-client-secret", os.Getenv("GDRIVE_CLIENT_SECRET"), "User name. Defaults to environment var GDRIVE_CLIENT_SECRET.")
|
||||
driveTokenFile = flag.String("drive-token-file", os.Getenv("GDRIVE_TOKEN_FILE"), "API key (password). Defaults to environment var GDRIVE_TOKEN_FILE.")
|
||||
driveAuthCode = flag.String("drive-auth-code", "", "Pass in when requested to make the drive token file.")
|
||||
driveFullList = flag.Bool("drive-full-list", true, "Use a full listing for directory list. More data but usually quicker.")
|
||||
driveAuthCode = flag.String("drive-auth-code", "", "Pass in when requested to make the drive token file.")
|
||||
driveFullList = flag.Bool("drive-full-list", true, "Use a full listing for directory list. More data but usually quicker.")
|
||||
)
|
||||
|
||||
// String converts this FsDrive to a string
|
||||
|
|
@ -142,13 +161,7 @@ func (f *FsDrive) String() string {
|
|||
|
||||
// parseParse parses a drive 'url'
|
||||
func parseDrivePath(path string) (root string, err error) {
|
||||
parts := Match.FindAllStringSubmatch(path, -1)
|
||||
if len(parts) != 1 || len(parts[0]) != 2 {
|
||||
err = fmt.Errorf("Couldn't parse drive url %q", path)
|
||||
} else {
|
||||
root = parts[0][1]
|
||||
root = strings.Trim(root, "/")
|
||||
}
|
||||
root = strings.Trim(root, "/")
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -222,26 +235,29 @@ func MakeNewToken(t *oauth.Transport) error {
|
|||
}
|
||||
|
||||
// NewFs contstructs an FsDrive from the path, container:path
|
||||
func NewFs(path string) (fs.Fs, error) {
|
||||
if *driveClientId == "" {
|
||||
return nil, errors.New("Need -drive-client-id or environmental variable GDRIVE_CLIENT_ID")
|
||||
func NewFs(name, path string) (fs.Fs, error) {
|
||||
clientId := fs.ConfigFile.MustValue(name, "client_id")
|
||||
if clientId == "" {
|
||||
return nil, errors.New("client_id not found")
|
||||
}
|
||||
if *driveClientSecret == "" {
|
||||
return nil, errors.New("Need -drive-client-secret or environmental variable GDRIVE_CLIENT_SECRET")
|
||||
clientSecret := fs.ConfigFile.MustValue(name, "client_secret")
|
||||
if clientSecret == "" {
|
||||
return nil, errors.New("client_secret not found")
|
||||
}
|
||||
if *driveTokenFile == "" {
|
||||
return nil, errors.New("Need -drive-token-file or environmental variable GDRIVE_TOKEN_FILE")
|
||||
tokenFile := fs.ConfigFile.MustValue(name, "token_file")
|
||||
if tokenFile == "" {
|
||||
return nil, errors.New("token-file not found")
|
||||
}
|
||||
|
||||
// Settings for authorization.
|
||||
var driveConfig = &oauth.Config{
|
||||
ClientId: *driveClientId,
|
||||
ClientSecret: *driveClientSecret,
|
||||
ClientId: clientId,
|
||||
ClientSecret: clientSecret,
|
||||
Scope: "https://www.googleapis.com/auth/drive",
|
||||
RedirectURL: "urn:ietf:wg:oauth:2.0:oob",
|
||||
AuthURL: "https://accounts.google.com/o/oauth2/auth",
|
||||
TokenURL: "https://accounts.google.com/o/oauth2/token",
|
||||
TokenCache: oauth.CacheFile(*driveTokenFile),
|
||||
TokenCache: oauth.CacheFile(tokenFile),
|
||||
}
|
||||
|
||||
root, err := parseDrivePath(path)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,260 @@
|
|||
// Read and write the config file
|
||||
package fs
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"os/user"
|
||||
"path"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Unknwon/goconfig"
|
||||
)
|
||||
|
||||
const (
|
||||
configFileName = ".rclone.conf"
|
||||
)
|
||||
|
||||
// Global
|
||||
var (
|
||||
// Config file
|
||||
ConfigFile *goconfig.ConfigFile
|
||||
// Config file path
|
||||
ConfigPath string
|
||||
// Global config
|
||||
Config = &ConfigInfo{}
|
||||
// Home directory
|
||||
HomeDir string
|
||||
// Flags
|
||||
verbose = flag.Bool("verbose", false, "Print lots more stuff")
|
||||
quiet = flag.Bool("quiet", false, "Print as little stuff as possible")
|
||||
modifyWindow = flag.Duration("modify-window", time.Nanosecond, "Max time diff to be considered the same")
|
||||
checkers = flag.Int("checkers", 8, "Number of checkers to run in parallel.")
|
||||
transfers = flag.Int("transfers", 4, "Number of file transfers to run in parallel.")
|
||||
)
|
||||
|
||||
// Filesystem config options
|
||||
type ConfigInfo struct {
|
||||
Verbose bool
|
||||
Quiet bool
|
||||
ModifyWindow time.Duration
|
||||
Checkers int
|
||||
Transfers int
|
||||
}
|
||||
|
||||
// Loads the config file
|
||||
func LoadConfig() {
|
||||
// Read some flags if set
|
||||
//
|
||||
// FIXME read these from the config file too
|
||||
Config.Verbose = *verbose
|
||||
Config.Quiet = *quiet
|
||||
Config.ModifyWindow = *modifyWindow
|
||||
Config.Checkers = *checkers
|
||||
Config.Transfers = *transfers
|
||||
|
||||
// Find users home directory
|
||||
usr, err := user.Current()
|
||||
if err != nil {
|
||||
log.Printf("Couldn't find home directory: %v", err)
|
||||
return
|
||||
}
|
||||
HomeDir = usr.HomeDir
|
||||
ConfigPath = path.Join(HomeDir, configFileName)
|
||||
|
||||
// Load configuration file.
|
||||
ConfigFile, err = goconfig.LoadConfigFile(ConfigPath)
|
||||
if err != nil {
|
||||
log.Printf("Failed to load config file %v - using defaults", ConfigPath)
|
||||
}
|
||||
}
|
||||
|
||||
// Save configuration file.
|
||||
func SaveConfig() {
|
||||
err := goconfig.SaveConfigFile(ConfigFile, ConfigPath)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to save config file: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Show an overview of the config file
|
||||
func ShowConfig() {
|
||||
remotes := ConfigFile.GetSectionList()
|
||||
sort.Strings(remotes)
|
||||
fmt.Printf("%-20s %s\n", "Name", "Type")
|
||||
fmt.Printf("%-20s %s\n", "====", "====")
|
||||
for _, remote := range remotes {
|
||||
fmt.Printf("%-20s %s\n", remote, ConfigFile.MustValue(remote, "type"))
|
||||
}
|
||||
}
|
||||
|
||||
// ChooseRemote chooses a remote name
|
||||
func ChooseRemote() string {
|
||||
remotes := ConfigFile.GetSectionList()
|
||||
sort.Strings(remotes)
|
||||
return Choose("remote", remotes, nil, false)
|
||||
}
|
||||
|
||||
// Read some input
|
||||
func ReadLine() string {
|
||||
buf := bufio.NewReader(os.Stdin)
|
||||
line, err := buf.ReadString('\n')
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to read line: %v", err)
|
||||
}
|
||||
return strings.TrimSpace(line)
|
||||
}
|
||||
|
||||
// Command - choose one
|
||||
func Command(commands []string) int {
|
||||
opts := []string{}
|
||||
for _, text := range commands {
|
||||
fmt.Printf("%c) %s\n", text[0], text[1:])
|
||||
opts = append(opts, text[:1])
|
||||
}
|
||||
optString := strings.Join(opts, "")
|
||||
optHelp := strings.Join(opts, "/")
|
||||
for {
|
||||
fmt.Printf("%s> ", optHelp)
|
||||
result := strings.ToLower(ReadLine())
|
||||
if len(result) != 1 {
|
||||
continue
|
||||
}
|
||||
i := strings.IndexByte(optString, result[0])
|
||||
if i >= 0 {
|
||||
return i
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Choose one of the defaults or type a new string if newOk is set
|
||||
func Choose(what string, defaults, help []string, newOk bool) string {
|
||||
fmt.Printf("Choose a number from below")
|
||||
if newOk {
|
||||
fmt.Printf(", or type in your own value")
|
||||
}
|
||||
fmt.Println()
|
||||
for i, text := range defaults {
|
||||
if help != nil {
|
||||
parts := strings.Split(help[i], "\n")
|
||||
for _, part := range parts {
|
||||
fmt.Printf(" * %s\n", part)
|
||||
}
|
||||
}
|
||||
fmt.Printf("%2d) %s\n", i+1, text)
|
||||
}
|
||||
for {
|
||||
fmt.Printf("%s> ", what)
|
||||
result := ReadLine()
|
||||
i, err := strconv.Atoi(result)
|
||||
if err != nil {
|
||||
if newOk {
|
||||
return result
|
||||
}
|
||||
continue
|
||||
}
|
||||
if i >= 1 && i <= len(defaults) {
|
||||
return defaults[i-1]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Show the contents of the remote
|
||||
func ShowRemote(name string) {
|
||||
fmt.Printf("--------------------\n")
|
||||
fmt.Printf("[%s]\n", name)
|
||||
for _, key := range ConfigFile.GetKeyList(name) {
|
||||
fmt.Printf("%s = %s\n", key, ConfigFile.MustValue(name, key))
|
||||
}
|
||||
fmt.Printf("--------------------\n")
|
||||
}
|
||||
|
||||
// Print the contents of the remote and ask if it is OK
|
||||
func OkRemote(name string) bool {
|
||||
ShowRemote(name)
|
||||
switch i := Command([]string{"yYes this is OK", "eEdit this remote", "dDelete this remote"}); i {
|
||||
case 0:
|
||||
return true
|
||||
case 1:
|
||||
return false
|
||||
case 2:
|
||||
ConfigFile.DeleteSection(name)
|
||||
return true
|
||||
default:
|
||||
log.Printf("Bad choice %d", i)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Make a new remote
|
||||
func NewRemote(name string) {
|
||||
fmt.Printf("What type of source is it?\n")
|
||||
types := []string{}
|
||||
for _, item := range fsRegistry {
|
||||
types = append(types, item.Name)
|
||||
}
|
||||
newType := Choose("type", types, nil, false)
|
||||
ConfigFile.SetValue(name, "type", newType)
|
||||
fs, err := Find(newType)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to find fs: %v", err)
|
||||
}
|
||||
for _, option := range fs.Options {
|
||||
ConfigFile.SetValue(name, option.Name, option.Choose())
|
||||
}
|
||||
if OkRemote(name) {
|
||||
SaveConfig()
|
||||
return
|
||||
}
|
||||
EditRemote(name)
|
||||
}
|
||||
|
||||
// Edit a remote
|
||||
func EditRemote(name string) {
|
||||
ShowRemote(name)
|
||||
fmt.Printf("Edit remote\n")
|
||||
for {
|
||||
for _, key := range ConfigFile.GetKeyList(name) {
|
||||
value := ConfigFile.MustValue(name, key)
|
||||
fmt.Printf("Press enter to accept current value, or type in a new one\n")
|
||||
fmt.Printf("%s = %s>", key, value)
|
||||
newValue := ReadLine()
|
||||
if newValue != "" {
|
||||
ConfigFile.SetValue(name, key, newValue)
|
||||
}
|
||||
}
|
||||
if OkRemote(name) {
|
||||
break
|
||||
}
|
||||
}
|
||||
SaveConfig()
|
||||
}
|
||||
|
||||
// Edit the config file interactively
|
||||
func EditConfig() {
|
||||
for {
|
||||
fmt.Printf("Current remotes:\n\n")
|
||||
ShowConfig()
|
||||
fmt.Printf("\n")
|
||||
switch i := Command([]string{"eEdit existing remote", "nNew remote", "dDelete remote", "qQuit config"}); i {
|
||||
case 0:
|
||||
name := ChooseRemote()
|
||||
EditRemote(name)
|
||||
case 1:
|
||||
fmt.Printf("name> ")
|
||||
name := ReadLine()
|
||||
NewRemote(name)
|
||||
case 2:
|
||||
name := ChooseRemote()
|
||||
ConfigFile.DeleteSection(name)
|
||||
case 3:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
99
fs/fs.go
99
fs/fs.go
|
|
@ -12,41 +12,52 @@ import (
|
|||
|
||||
// Globals
|
||||
var (
|
||||
// Global config
|
||||
Config = &ConfigInfo{}
|
||||
// Filesystem registry
|
||||
fsRegistry []registryItem
|
||||
fsRegistry []*FsInfo
|
||||
)
|
||||
|
||||
// Filesystem config options
|
||||
type ConfigInfo struct {
|
||||
Verbose bool
|
||||
Quiet bool
|
||||
ModifyWindow time.Duration
|
||||
Checkers int
|
||||
Transfers int
|
||||
// Filesystem info
|
||||
type FsInfo struct {
|
||||
Name string // name of this fs
|
||||
NewFs func(string, string) (Fs, error) // create a new file system
|
||||
Options []Option
|
||||
}
|
||||
|
||||
// Filesystem registry item
|
||||
type registryItem struct {
|
||||
match *regexp.Regexp // if this matches then can call newFs
|
||||
newFs func(string) (Fs, error) // create a new file system
|
||||
// An options for a Fs
|
||||
type Option struct {
|
||||
Name string
|
||||
Help string
|
||||
Optional bool
|
||||
Examples []OptionExample
|
||||
}
|
||||
|
||||
// An example for an option
|
||||
type OptionExample struct {
|
||||
Value string
|
||||
Help string
|
||||
}
|
||||
|
||||
// Choose an option
|
||||
func (o *Option) Choose() string {
|
||||
fmt.Println(o.Help)
|
||||
if len(o.Examples) > 0 {
|
||||
var values []string
|
||||
var help []string
|
||||
for _, example := range o.Examples {
|
||||
values = append(values, example.Value)
|
||||
help = append(help, example.Help)
|
||||
}
|
||||
return Choose(o.Name, values, help, true)
|
||||
}
|
||||
fmt.Printf("%s> ", o.Name)
|
||||
return ReadLine()
|
||||
}
|
||||
|
||||
// Register a filesystem
|
||||
//
|
||||
// If a path matches with match then can call newFs on it
|
||||
//
|
||||
// Pass with match nil goes last and matches everything (used by local fs)
|
||||
//
|
||||
// Fs modules should use this in an init() function
|
||||
func Register(match *regexp.Regexp, newFs func(string) (Fs, error)) {
|
||||
fsRegistry = append(fsRegistry, registryItem{match: match, newFs: newFs})
|
||||
// Keep one nil match at the end
|
||||
last := len(fsRegistry) - 1
|
||||
if last >= 1 && fsRegistry[last-1].match == nil {
|
||||
fsRegistry[last], fsRegistry[last-1] = fsRegistry[last-1], fsRegistry[last]
|
||||
}
|
||||
func Register(info *FsInfo) {
|
||||
fsRegistry = append(fsRegistry, info)
|
||||
}
|
||||
|
||||
// A Filesystem, describes the local filesystem and the remote object store
|
||||
|
|
@ -136,16 +147,42 @@ type Dir struct {
|
|||
// A channel of Dir objects
|
||||
type DirChan chan *Dir
|
||||
|
||||
// NewFs makes a new Fs object from the path
|
||||
// Pattern to match a url
|
||||
var matcher = regexp.MustCompile(`^([\w_-]+)://(.*)$`)
|
||||
|
||||
// Finds a FsInfo object for the name passed in
|
||||
//
|
||||
// FIXME make more generic
|
||||
func NewFs(path string) (Fs, error) {
|
||||
// Services are looked up in the config file
|
||||
func Find(name string) (*FsInfo, error) {
|
||||
for _, item := range fsRegistry {
|
||||
if item.match == nil || item.match.MatchString(path) {
|
||||
return item.newFs(path)
|
||||
if item.Name == name {
|
||||
return item, nil
|
||||
}
|
||||
}
|
||||
panic("Not found") // FIXME
|
||||
return nil, fmt.Errorf("Didn't find filing system for %q", name)
|
||||
}
|
||||
|
||||
// NewFs makes a new Fs object from the path
|
||||
//
|
||||
// The path is of the form service://path
|
||||
//
|
||||
// Services are looked up in the config file
|
||||
func NewFs(path string) (Fs, error) {
|
||||
parts := matcher.FindStringSubmatch(path)
|
||||
fsName, configName, fsPath := "local", "local", path
|
||||
if parts != nil {
|
||||
configName, fsPath = parts[1], parts[2]
|
||||
var err error
|
||||
fsName, err = ConfigFile.GetValue(configName, "type")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Didn't find section in config file for %q", configName)
|
||||
}
|
||||
}
|
||||
fs, err := Find(fsName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return fs.NewFs(configName, fsPath)
|
||||
}
|
||||
|
||||
// Write debuging output for this Object
|
||||
|
|
|
|||
10
local/fs.go
10
local/fs.go
|
|
@ -4,7 +4,6 @@ package local
|
|||
import (
|
||||
"crypto/md5"
|
||||
"fmt"
|
||||
"github.com/ncw/rclone/fs"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
|
|
@ -13,11 +12,16 @@ import (
|
|||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/ncw/rclone/fs"
|
||||
)
|
||||
|
||||
// Register with Fs
|
||||
func init() {
|
||||
fs.Register(nil, NewFs)
|
||||
fs.Register(&fs.FsInfo{
|
||||
Name: "local",
|
||||
NewFs: NewFs,
|
||||
})
|
||||
}
|
||||
|
||||
// FsLocal represents a local filesystem rooted at root
|
||||
|
|
@ -37,7 +41,7 @@ type FsObjectLocal struct {
|
|||
// ------------------------------------------------------------
|
||||
|
||||
// NewFs contstructs an FsLocal from the path
|
||||
func NewFs(root string) (fs.Fs, error) {
|
||||
func NewFs(name, root string) (fs.Fs, error) {
|
||||
root = path.Clean(root)
|
||||
f := &FsLocal{root: root}
|
||||
return f, nil
|
||||
|
|
|
|||
236
rclone.go
236
rclone.go
|
|
@ -6,7 +6,6 @@ package main
|
|||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"github.com/ncw/rclone/fs"
|
||||
"log"
|
||||
"os"
|
||||
"runtime"
|
||||
|
|
@ -14,6 +13,8 @@ import (
|
|||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/ncw/rclone/fs"
|
||||
// Active file systems
|
||||
_ "github.com/ncw/rclone/drive"
|
||||
_ "github.com/ncw/rclone/local"
|
||||
|
|
@ -25,13 +26,8 @@ import (
|
|||
var (
|
||||
// Flags
|
||||
cpuprofile = flag.String("cpuprofile", "", "Write cpu profile to file")
|
||||
verbose = flag.Bool("verbose", false, "Print lots more stuff")
|
||||
quiet = flag.Bool("quiet", false, "Print as little stuff as possible")
|
||||
dry_run = flag.Bool("dry-run", false, "Do a trial run with no permanent changes")
|
||||
checkers = flag.Int("checkers", 8, "Number of checkers to run in parallel.")
|
||||
transfers = flag.Int("transfers", 4, "Number of file transfers to run in parallel.")
|
||||
statsInterval = flag.Duration("stats", time.Minute*1, "Interval to print stats")
|
||||
modifyWindow = flag.Duration("modify-window", time.Nanosecond, "Max time diff to be considered the same")
|
||||
)
|
||||
|
||||
// A pair of fs.Objects
|
||||
|
|
@ -105,17 +101,17 @@ func CopyFs(fdst, fsrc fs.Fs) {
|
|||
}
|
||||
|
||||
to_be_checked := fsrc.List()
|
||||
to_be_uploaded := make(fs.ObjectsChan, *transfers)
|
||||
to_be_uploaded := make(fs.ObjectsChan, fs.Config.Transfers)
|
||||
|
||||
var checkerWg sync.WaitGroup
|
||||
checkerWg.Add(*checkers)
|
||||
for i := 0; i < *checkers; i++ {
|
||||
checkerWg.Add(fs.Config.Checkers)
|
||||
for i := 0; i < fs.Config.Checkers; i++ {
|
||||
go Checker(to_be_checked, to_be_uploaded, fdst, &checkerWg)
|
||||
}
|
||||
|
||||
var copierWg sync.WaitGroup
|
||||
copierWg.Add(*transfers)
|
||||
for i := 0; i < *transfers; i++ {
|
||||
copierWg.Add(fs.Config.Transfers)
|
||||
for i := 0; i < fs.Config.Transfers; i++ {
|
||||
go Copier(to_be_uploaded, fdst, &copierWg)
|
||||
}
|
||||
|
||||
|
|
@ -129,8 +125,8 @@ func CopyFs(fdst, fsrc fs.Fs) {
|
|||
// Delete all the files passed in the channel
|
||||
func DeleteFiles(to_be_deleted fs.ObjectsChan) {
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(*transfers)
|
||||
for i := 0; i < *transfers; i++ {
|
||||
wg.Add(fs.Config.Transfers)
|
||||
for i := 0; i < fs.Config.Transfers; i++ {
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for dst := range to_be_deleted {
|
||||
|
|
@ -173,18 +169,18 @@ func Sync(fdst, fsrc fs.Fs) {
|
|||
}
|
||||
|
||||
// Read source files checking them off against dest files
|
||||
to_be_checked := make(PairFsObjectsChan, *transfers)
|
||||
to_be_uploaded := make(fs.ObjectsChan, *transfers)
|
||||
to_be_checked := make(PairFsObjectsChan, fs.Config.Transfers)
|
||||
to_be_uploaded := make(fs.ObjectsChan, fs.Config.Transfers)
|
||||
|
||||
var checkerWg sync.WaitGroup
|
||||
checkerWg.Add(*checkers)
|
||||
for i := 0; i < *checkers; i++ {
|
||||
checkerWg.Add(fs.Config.Checkers)
|
||||
for i := 0; i < fs.Config.Checkers; i++ {
|
||||
go PairChecker(to_be_checked, to_be_uploaded, &checkerWg)
|
||||
}
|
||||
|
||||
var copierWg sync.WaitGroup
|
||||
copierWg.Add(*transfers)
|
||||
for i := 0; i < *transfers; i++ {
|
||||
copierWg.Add(fs.Config.Transfers)
|
||||
for i := 0; i < fs.Config.Transfers; i++ {
|
||||
go Copier(to_be_uploaded, fdst, &copierWg)
|
||||
}
|
||||
|
||||
|
|
@ -215,7 +211,7 @@ func Sync(fdst, fsrc fs.Fs) {
|
|||
}
|
||||
|
||||
// Delete the spare files
|
||||
toDelete := make(fs.ObjectsChan, *transfers)
|
||||
toDelete := make(fs.ObjectsChan, fs.Config.Transfers)
|
||||
go func() {
|
||||
for _, fs := range delFiles {
|
||||
toDelete <- fs
|
||||
|
|
@ -262,7 +258,7 @@ func Check(fdst, fsrc fs.Fs) {
|
|||
log.Printf(remote)
|
||||
}
|
||||
|
||||
checks := make(chan []fs.Object, *transfers)
|
||||
checks := make(chan []fs.Object, fs.Config.Transfers)
|
||||
go func() {
|
||||
for _, check := range commonFiles {
|
||||
checks <- check
|
||||
|
|
@ -271,8 +267,8 @@ func Check(fdst, fsrc fs.Fs) {
|
|||
}()
|
||||
|
||||
var checkerWg sync.WaitGroup
|
||||
checkerWg.Add(*checkers)
|
||||
for i := 0; i < *checkers; i++ {
|
||||
checkerWg.Add(fs.Config.Checkers)
|
||||
for i := 0; i < fs.Config.Checkers; i++ {
|
||||
go func() {
|
||||
defer checkerWg.Done()
|
||||
for check := range checks {
|
||||
|
|
@ -309,8 +305,8 @@ func Check(fdst, fsrc fs.Fs) {
|
|||
func List(f, _ fs.Fs) {
|
||||
in := f.List()
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(*checkers)
|
||||
for i := 0; i < *checkers; i++ {
|
||||
wg.Add(fs.Config.Checkers)
|
||||
for i := 0; i < fs.Config.Checkers; i++ {
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for o := range in {
|
||||
|
|
@ -370,117 +366,128 @@ func purge(fdst, fsrc fs.Fs) {
|
|||
}
|
||||
}
|
||||
|
||||
// Edits the config file
|
||||
func EditConfig(fdst, fsrc fs.Fs) {
|
||||
fs.EditConfig()
|
||||
}
|
||||
|
||||
type Command struct {
|
||||
name string
|
||||
help string
|
||||
run func(fdst, fsrc fs.Fs)
|
||||
minArgs, maxArgs int
|
||||
Name string
|
||||
Help string
|
||||
ArgsHelp string
|
||||
Run func(fdst, fsrc fs.Fs)
|
||||
MinArgs int
|
||||
MaxArgs int
|
||||
NoStats bool
|
||||
}
|
||||
|
||||
// checkArgs checks there are enough arguments and prints a message if not
|
||||
func (cmd *Command) checkArgs(args []string) {
|
||||
if len(args) < cmd.minArgs {
|
||||
if len(args) < cmd.MinArgs {
|
||||
syntaxError()
|
||||
fmt.Fprintf(os.Stderr, "Command %s needs %d arguments mininum\n", cmd.name, cmd.minArgs)
|
||||
fmt.Fprintf(os.Stderr, "Command %s needs %d arguments mininum\n", cmd.Name, cmd.MinArgs)
|
||||
os.Exit(1)
|
||||
} else if len(args) > cmd.maxArgs {
|
||||
} else if len(args) > cmd.MaxArgs {
|
||||
syntaxError()
|
||||
fmt.Fprintf(os.Stderr, "Command %s needs %d arguments maximum\n", cmd.name, cmd.maxArgs)
|
||||
fmt.Fprintf(os.Stderr, "Command %s needs %d arguments maximum\n", cmd.Name, cmd.MaxArgs)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
var Commands = []Command{
|
||||
{
|
||||
"copy",
|
||||
`<source> <destination>
|
||||
|
||||
Name: "copy",
|
||||
ArgsHelp: "source://path dest://path",
|
||||
Help: `
|
||||
Copy the source to the destination. Doesn't transfer
|
||||
unchanged files, testing first by modification time then by
|
||||
MD5SUM. Doesn't delete files from the destination.
|
||||
|
||||
`,
|
||||
CopyFs,
|
||||
2, 2,
|
||||
MD5SUM. Doesn't delete files from the destination.`,
|
||||
Run: CopyFs,
|
||||
MinArgs: 2,
|
||||
MaxArgs: 2,
|
||||
},
|
||||
{
|
||||
"sync",
|
||||
`<source> <destination>
|
||||
|
||||
Name: "sync",
|
||||
ArgsHelp: "source://path dest://path",
|
||||
Help: `
|
||||
Sync the source to the destination. Doesn't transfer
|
||||
unchanged files, testing first by modification time then by
|
||||
MD5SUM. Deletes any files that exist in source that don't
|
||||
exist in destination. Since this can cause data loss, test
|
||||
first with the -dry-run flag.`,
|
||||
|
||||
Sync,
|
||||
2, 2,
|
||||
Run: Sync,
|
||||
MinArgs: 2,
|
||||
MaxArgs: 2,
|
||||
},
|
||||
{
|
||||
"ls",
|
||||
`[<path>]
|
||||
|
||||
Name: "ls",
|
||||
ArgsHelp: "[remote://path]",
|
||||
Help: `
|
||||
List all the objects in the the path.`,
|
||||
|
||||
List,
|
||||
1, 1,
|
||||
Run: List,
|
||||
MinArgs: 1,
|
||||
MaxArgs: 1,
|
||||
},
|
||||
{
|
||||
"lsd",
|
||||
`[<path>]
|
||||
|
||||
Name: "lsd",
|
||||
ArgsHelp: "[remote://path]",
|
||||
Help: `
|
||||
List all directoryes/objects/buckets in the the path.`,
|
||||
|
||||
ListDir,
|
||||
1, 1,
|
||||
Run: ListDir,
|
||||
MinArgs: 1,
|
||||
MaxArgs: 1,
|
||||
},
|
||||
{
|
||||
"mkdir",
|
||||
`<path>
|
||||
|
||||
Name: "mkdir",
|
||||
ArgsHelp: "remote://path",
|
||||
Help: `
|
||||
Make the path if it doesn't already exist`,
|
||||
|
||||
mkdir,
|
||||
1, 1,
|
||||
Run: mkdir,
|
||||
MinArgs: 1,
|
||||
MaxArgs: 1,
|
||||
},
|
||||
{
|
||||
"rmdir",
|
||||
`<path>
|
||||
|
||||
Name: "rmdir",
|
||||
ArgsHelp: "remote://path",
|
||||
Help: `
|
||||
Remove the path. Note that you can't remove a path with
|
||||
objects in it, use purge for that.`,
|
||||
|
||||
rmdir,
|
||||
1, 1,
|
||||
Run: rmdir,
|
||||
MinArgs: 1,
|
||||
MaxArgs: 1,
|
||||
},
|
||||
{
|
||||
"purge",
|
||||
`<path>
|
||||
|
||||
Name: "purge",
|
||||
ArgsHelp: "remote://path",
|
||||
Help: `
|
||||
Remove the path and all of its contents.`,
|
||||
|
||||
purge,
|
||||
1, 1,
|
||||
Run: purge,
|
||||
MinArgs: 1,
|
||||
MaxArgs: 1,
|
||||
},
|
||||
{
|
||||
"check",
|
||||
`<source> <destination>
|
||||
|
||||
Name: "check",
|
||||
ArgsHelp: "source://path dest://path",
|
||||
Help: `
|
||||
Checks the files in the source and destination match. It
|
||||
compares sizes and MD5SUMs and prints a report of files which
|
||||
don't match. It doesn't alter the source or destination.`,
|
||||
|
||||
Check,
|
||||
2, 2,
|
||||
Run: Check,
|
||||
MinArgs: 2,
|
||||
MaxArgs: 2,
|
||||
},
|
||||
{
|
||||
"help",
|
||||
`
|
||||
|
||||
Name: "config",
|
||||
Help: `
|
||||
Enter an interactive configuration session.`,
|
||||
Run: EditConfig,
|
||||
NoStats: true,
|
||||
},
|
||||
{
|
||||
Name: "help",
|
||||
Help: `
|
||||
This help.`,
|
||||
|
||||
nil,
|
||||
0, 0,
|
||||
NoStats: true,
|
||||
},
|
||||
}
|
||||
|
||||
|
|
@ -495,7 +502,8 @@ Subcommands:
|
|||
`)
|
||||
for i := range Commands {
|
||||
cmd := &Commands[i]
|
||||
fmt.Fprintf(os.Stderr, " %s: %s\n\n", cmd.name, cmd.help)
|
||||
fmt.Fprintf(os.Stderr, " %s %s\n", cmd.Name, cmd.ArgsHelp)
|
||||
fmt.Fprintf(os.Stderr, "%s\n\n", cmd.Help)
|
||||
}
|
||||
|
||||
fmt.Fprintf(os.Stderr, "Options:\n")
|
||||
|
|
@ -517,13 +525,7 @@ func main() {
|
|||
flag.Parse()
|
||||
args := flag.Args()
|
||||
runtime.GOMAXPROCS(runtime.NumCPU())
|
||||
|
||||
// Pass on some flags to fs.Config
|
||||
fs.Config.Verbose = *verbose
|
||||
fs.Config.Quiet = *quiet
|
||||
fs.Config.ModifyWindow = *modifyWindow
|
||||
fs.Config.Checkers = *checkers
|
||||
fs.Config.Transfers = *transfers
|
||||
fs.LoadConfig()
|
||||
|
||||
// Setup profiling if desired
|
||||
if *cpuprofile != "" {
|
||||
|
|
@ -548,10 +550,10 @@ func main() {
|
|||
for i := range Commands {
|
||||
command := &Commands[i]
|
||||
// exact command name found - use that
|
||||
if command.name == cmd {
|
||||
if command.Name == cmd {
|
||||
found = command
|
||||
break
|
||||
} else if strings.HasPrefix(command.name, cmd) {
|
||||
} else if strings.HasPrefix(command.Name, cmd) {
|
||||
if found != nil {
|
||||
fs.Stats.Error()
|
||||
log.Fatalf("Not unique - matches multiple commands %q", cmd)
|
||||
|
|
@ -572,14 +574,14 @@ func main() {
|
|||
fdst, err = fs.NewFs(args[0])
|
||||
if err != nil {
|
||||
fs.Stats.Error()
|
||||
log.Fatal("Failed to create file system: ", err)
|
||||
log.Fatalf("Failed to create file system for %q: %v", args[0], err)
|
||||
}
|
||||
}
|
||||
if len(args) >= 2 {
|
||||
fsrc, err = fs.NewFs(args[1])
|
||||
if err != nil {
|
||||
fs.Stats.Error()
|
||||
log.Fatal("Failed to create destination file system: ", err)
|
||||
log.Fatalf("Failed to create destination file system for %q: %v", args[1], err)
|
||||
}
|
||||
fsrc, fdst = fdst, fsrc
|
||||
}
|
||||
|
|
@ -599,22 +601,30 @@ func main() {
|
|||
fs.Config.ModifyWindow = precision
|
||||
}
|
||||
}
|
||||
log.Printf("Modify window is %s\n", fs.Config.ModifyWindow)
|
||||
if fs.Config.Verbose {
|
||||
log.Printf("Modify window is %s\n", fs.Config.ModifyWindow)
|
||||
}
|
||||
|
||||
// Print the stats every statsInterval
|
||||
go func() {
|
||||
ch := time.Tick(*statsInterval)
|
||||
for {
|
||||
<-ch
|
||||
fs.Stats.Log()
|
||||
}
|
||||
}()
|
||||
if !found.NoStats {
|
||||
go func() {
|
||||
ch := time.Tick(*statsInterval)
|
||||
for {
|
||||
<-ch
|
||||
fs.Stats.Log()
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Run the actual command
|
||||
if found.run != nil {
|
||||
found.run(fdst, fsrc)
|
||||
fmt.Println(fs.Stats)
|
||||
log.Printf("*** Go routines at exit %d\n", runtime.NumGoroutine())
|
||||
if found.Run != nil {
|
||||
found.Run(fdst, fsrc)
|
||||
if !found.NoStats {
|
||||
fmt.Println(fs.Stats)
|
||||
}
|
||||
if fs.Config.Verbose {
|
||||
log.Printf("*** Go routines at exit %d\n", runtime.NumGoroutine())
|
||||
}
|
||||
if fs.Stats.Errored() {
|
||||
os.Exit(1)
|
||||
}
|
||||
|
|
|
|||
140
s3/fs.go
140
s3/fs.go
|
|
@ -5,30 +5,99 @@ package s3
|
|||
|
||||
import (
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"github.com/ncw/goamz/aws"
|
||||
"github.com/ncw/goamz/s3"
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/ncw/swift"
|
||||
"io"
|
||||
"log"
|
||||
"mime"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Pattern to match a s3 url
|
||||
var Match = regexp.MustCompile(`^s3://([^/]*)(.*)$`)
|
||||
"github.com/ncw/goamz/aws"
|
||||
"github.com/ncw/goamz/s3"
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/ncw/swift"
|
||||
)
|
||||
|
||||
// Register with Fs
|
||||
func init() {
|
||||
fs.Register(Match, NewFs)
|
||||
fs.Register(&fs.FsInfo{
|
||||
Name: "s3",
|
||||
NewFs: NewFs,
|
||||
// AWS endpoints: http://docs.amazonwebservices.com/general/latest/gr/rande.html#s3_region
|
||||
Options: []fs.Option{{
|
||||
Name: "access_key_id",
|
||||
Help: "AWS Access Key ID.",
|
||||
}, {
|
||||
Name: "secret_access_key",
|
||||
Help: "AWS Secret Access Key (password). ",
|
||||
}, {
|
||||
Name: "endpoint",
|
||||
Help: "Endpoint for S3 API.",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "https://s3.amazonaws.com/",
|
||||
Help: "The default endpoint - a good choice if you are unsure.\nUS Region, Northern Virginia or Pacific Northwest.\nLeave location constraint empty.",
|
||||
}, {
|
||||
Value: "https://s3-external-1.amazonaws.com",
|
||||
Help: "US Region, Northern Virginia only.\nLeave location constraint empty.",
|
||||
}, {
|
||||
Value: "https://s3-us-west-2.amazonaws.com",
|
||||
Help: "US West (Oregon) Region\nNeeds location constraint us-west-2.",
|
||||
}, {
|
||||
Value: "https://s3-us-west-1.amazonaws.com",
|
||||
Help: "US West (Northern California) Region\nNeeds location constraint us-west-1.",
|
||||
}, {
|
||||
Value: "https://s3-eu-west-1.amazonaws.com",
|
||||
Help: "EU (Ireland) Region Region\nNeeds location constraint EU or eu-west-1.",
|
||||
}, {
|
||||
Value: "https://s3-ap-southeast-1.amazonaws.com",
|
||||
Help: "Asia Pacific (Singapore) Region\nNeeds location constraint ap-southeast-1.",
|
||||
}, {
|
||||
Value: "https://s3-ap-southeast-2.amazonaws.com",
|
||||
Help: "Asia Pacific (Sydney) Region\nNeeds location constraint .",
|
||||
}, {
|
||||
Value: "https://s3-ap-northeast-1.amazonaws.com",
|
||||
Help: "Asia Pacific (Tokyo) Region\nNeeds location constraint ap-northeast-1.",
|
||||
}, {
|
||||
Value: "https://s3-sa-east-1.amazonaws.com",
|
||||
Help: "South America (Sao Paulo) Region\nNeeds location constraint sa-east-1.",
|
||||
}},
|
||||
}, {
|
||||
Name: "location_constraint",
|
||||
Help: "Location constraint - must be set to match the Endpoint.",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "",
|
||||
Help: "Empty for US Region, Northern Virginia or Pacific Northwest.",
|
||||
}, {
|
||||
Value: "us-west-2",
|
||||
Help: "US West (Oregon) Region.",
|
||||
}, {
|
||||
Value: "us-west-1",
|
||||
Help: "US West (Northern California) Region.",
|
||||
}, {
|
||||
Value: "eu-west-1",
|
||||
Help: "EU (Ireland) Region.",
|
||||
}, {
|
||||
Value: "EU",
|
||||
Help: "EU Region.",
|
||||
}, {
|
||||
Value: "ap-southeast-1",
|
||||
Help: "Asia Pacific (Singapore) Region.",
|
||||
}, {
|
||||
Value: "ap-southeast-2",
|
||||
Help: "Asia Pacific (Sydney) Region.",
|
||||
}, {
|
||||
Value: "ap-northeast-1",
|
||||
Help: "Asia Pacific (Tokyo) Region.",
|
||||
}, {
|
||||
Value: "sa-east-1",
|
||||
Help: "South America (Sao Paulo) Region.",
|
||||
}},
|
||||
}},
|
||||
})
|
||||
}
|
||||
|
||||
// Constants
|
||||
|
|
@ -60,57 +129,54 @@ type FsObjectS3 struct {
|
|||
|
||||
// ------------------------------------------------------------
|
||||
|
||||
// Globals
|
||||
var (
|
||||
// Flags
|
||||
awsAccessKeyId = flag.String("aws-access-key-id", os.Getenv("AWS_ACCESS_KEY_ID"), "AWS Access Key ID. Defaults to environment var AWS_ACCESS_KEY_ID.")
|
||||
awsSecretAccessKey = flag.String("aws-secret-access-key", os.Getenv("AWS_SECRET_ACCESS_KEY"), "AWS Secret Access Key (password). Defaults to environment var AWS_SECRET_ACCESS_KEY.")
|
||||
// AWS endpoints: http://docs.amazonwebservices.com/general/latest/gr/rande.html#s3_region
|
||||
s3Endpoint = flag.String("s3-endpoint", os.Getenv("S3_ENDPOINT"), "S3 Endpoint. Defaults to environment var S3_ENDPOINT then https://s3.amazonaws.com/.")
|
||||
s3LocationConstraint = flag.String("s3-location-constraint", os.Getenv("S3_LOCATION_CONSTRAINT"), "Location constraint for creating buckets only. Defaults to environment var S3_LOCATION_CONSTRAINT.")
|
||||
)
|
||||
|
||||
// String converts this FsS3 to a string
|
||||
func (f *FsS3) String() string {
|
||||
return fmt.Sprintf("S3 bucket %s", f.bucket)
|
||||
}
|
||||
|
||||
// Pattern to match a s3 path
|
||||
var matcher = regexp.MustCompile(`^([^/]*)(.*)$`)
|
||||
|
||||
// parseParse parses a s3 'url'
|
||||
func s3ParsePath(path string) (bucket, directory string, err error) {
|
||||
parts := Match.FindAllStringSubmatch(path, -1)
|
||||
if len(parts) != 1 || len(parts[0]) != 3 {
|
||||
err = fmt.Errorf("Couldn't parse s3 url %q", path)
|
||||
parts := matcher.FindStringSubmatch(path)
|
||||
if parts == nil {
|
||||
err = fmt.Errorf("Couldn't parse bucket out of s3 path %q", path)
|
||||
} else {
|
||||
bucket, directory = parts[0][1], parts[0][2]
|
||||
bucket, directory = parts[1], parts[2]
|
||||
directory = strings.Trim(directory, "/")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// s3Connection makes a connection to s3
|
||||
func s3Connection() (*s3.S3, error) {
|
||||
func s3Connection(name string) (*s3.S3, error) {
|
||||
// Make the auth
|
||||
if *awsAccessKeyId == "" {
|
||||
return nil, errors.New("Need -aws-access-key-id or environmental variable AWS_ACCESS_KEY_ID")
|
||||
accessKeyId := fs.ConfigFile.MustValue(name, "access_key_id")
|
||||
if accessKeyId == "" {
|
||||
return nil, errors.New("access_key_id not found")
|
||||
}
|
||||
if *awsSecretAccessKey == "" {
|
||||
return nil, errors.New("Need -aws-secret-access-key or environmental variable AWS_SECRET_ACCESS_KEY")
|
||||
secretAccessKey := fs.ConfigFile.MustValue(name, "secret_access_key")
|
||||
if secretAccessKey == "" {
|
||||
return nil, errors.New("secret_access_key not found")
|
||||
}
|
||||
auth := aws.Auth{AccessKey: *awsAccessKeyId, SecretKey: *awsSecretAccessKey}
|
||||
auth := aws.Auth{AccessKey: accessKeyId, SecretKey: secretAccessKey}
|
||||
|
||||
// FIXME look through all the regions by name and use one of them if found
|
||||
|
||||
// Synthesize the region
|
||||
if *s3Endpoint == "" {
|
||||
*s3Endpoint = "https://s3.amazonaws.com/"
|
||||
s3Endpoint := fs.ConfigFile.MustValue(name, "endpoint")
|
||||
if s3Endpoint == "" {
|
||||
s3Endpoint = "https://s3.amazonaws.com/"
|
||||
}
|
||||
region := aws.Region{
|
||||
Name: "s3",
|
||||
S3Endpoint: *s3Endpoint,
|
||||
S3Endpoint: s3Endpoint,
|
||||
S3LocationConstraint: false,
|
||||
}
|
||||
if *s3LocationConstraint != "" {
|
||||
region.Name = *s3LocationConstraint
|
||||
s3LocationConstraint := fs.ConfigFile.MustValue(name, "location_constraint")
|
||||
if s3LocationConstraint != "" {
|
||||
region.Name = s3LocationConstraint
|
||||
region.S3LocationConstraint = true
|
||||
}
|
||||
|
||||
|
|
@ -119,7 +185,7 @@ func s3Connection() (*s3.S3, error) {
|
|||
}
|
||||
|
||||
// NewFsS3 contstructs an FsS3 from the path, bucket:path
|
||||
func NewFs(path string) (fs.Fs, error) {
|
||||
func NewFs(name, path string) (fs.Fs, error) {
|
||||
bucket, directory, err := s3ParsePath(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
@ -127,7 +193,7 @@ func NewFs(path string) (fs.Fs, error) {
|
|||
if directory != "" {
|
||||
return nil, fmt.Errorf("Directories not supported yet in %q: %q", path, directory)
|
||||
}
|
||||
c, err := s3Connection()
|
||||
c, err := s3Connection(name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
|
|||
91
swift/fs.go
91
swift/fs.go
|
|
@ -5,24 +5,51 @@ package swift
|
|||
|
||||
import (
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/ncw/swift"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Pattern to match a swift url
|
||||
var Match = regexp.MustCompile(`^swift://([^/]*)(.*)$`)
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/ncw/swift"
|
||||
)
|
||||
|
||||
// Register with Fs
|
||||
func init() {
|
||||
fs.Register(Match, NewFs)
|
||||
fs.Register(&fs.FsInfo{
|
||||
Name: "swift",
|
||||
NewFs: NewFs,
|
||||
Options: []fs.Option{{
|
||||
Name: "user",
|
||||
Help: "User name to log in.",
|
||||
}, {
|
||||
Name: "key",
|
||||
Help: "API key or password.",
|
||||
}, {
|
||||
Name: "auth",
|
||||
Help: "Authentication URL for server.",
|
||||
Examples: []fs.OptionExample{{
|
||||
Help: "Rackspace US",
|
||||
Value: "https://auth.api.rackspacecloud.com/v1.0",
|
||||
}, {
|
||||
Help: "Rackspace UK",
|
||||
Value: "https://lon.auth.api.rackspacecloud.com/v1.0",
|
||||
}, {
|
||||
Help: "Rackspace v2",
|
||||
Value: "https://identity.api.rackspacecloud.com/v2.0",
|
||||
}, {
|
||||
Help: "Memset Memstore UK",
|
||||
Value: "https://auth.storage.memset.com/v1.0",
|
||||
}, {
|
||||
Help: "Memset Memstore UK v2",
|
||||
Value: "https://auth.storage.memset.com/v2.0",
|
||||
}},
|
||||
},
|
||||
// snet = flag.Bool("swift-snet", false, "Use internal service network") // FIXME not implemented
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// FsSwift represents a remote swift server
|
||||
|
|
@ -44,48 +71,44 @@ type FsObjectSwift struct {
|
|||
|
||||
// ------------------------------------------------------------
|
||||
|
||||
// Globals
|
||||
var (
|
||||
// Flags
|
||||
// FIXME make these part of swift so we get a standard set of flags?
|
||||
authUrl = flag.String("swift-auth", os.Getenv("ST_AUTH"), "Auth URL for server. Defaults to environment var ST_AUTH.")
|
||||
userName = flag.String("swift-user", os.Getenv("ST_USER"), "User name. Defaults to environment var ST_USER.")
|
||||
apiKey = flag.String("swift-key", os.Getenv("ST_KEY"), "API key (password). Defaults to environment var ST_KEY.")
|
||||
snet = flag.Bool("swift-snet", false, "Use internal service network") // FIXME not implemented
|
||||
)
|
||||
|
||||
// String converts this FsSwift to a string
|
||||
func (f *FsSwift) String() string {
|
||||
return fmt.Sprintf("Swift container %s", f.container)
|
||||
}
|
||||
|
||||
// Pattern to match a swift path
|
||||
var matcher = regexp.MustCompile(`^([^/]*)(.*)$`)
|
||||
|
||||
// parseParse parses a swift 'url'
|
||||
func parsePath(path string) (container, directory string, err error) {
|
||||
parts := Match.FindAllStringSubmatch(path, -1)
|
||||
if len(parts) != 1 || len(parts[0]) != 3 {
|
||||
err = fmt.Errorf("Couldn't parse swift url %q", path)
|
||||
parts := matcher.FindStringSubmatch(path)
|
||||
if parts == nil {
|
||||
err = fmt.Errorf("Couldn't find container in swift path %q", path)
|
||||
} else {
|
||||
container, directory = parts[0][1], parts[0][2]
|
||||
container, directory = parts[1], parts[2]
|
||||
directory = strings.Trim(directory, "/")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// swiftConnection makes a connection to swift
|
||||
func swiftConnection() (*swift.Connection, error) {
|
||||
if *userName == "" {
|
||||
return nil, errors.New("Need -user or environmental variable ST_USER")
|
||||
func swiftConnection(name string) (*swift.Connection, error) {
|
||||
userName := fs.ConfigFile.MustValue(name, "user")
|
||||
if userName == "" {
|
||||
return nil, errors.New("user not found")
|
||||
}
|
||||
if *apiKey == "" {
|
||||
return nil, errors.New("Need -key or environmental variable ST_KEY")
|
||||
apiKey := fs.ConfigFile.MustValue(name, "key")
|
||||
if apiKey == "" {
|
||||
return nil, errors.New("key not found")
|
||||
}
|
||||
if *authUrl == "" {
|
||||
return nil, errors.New("Need -auth or environmental variable ST_AUTH")
|
||||
authUrl := fs.ConfigFile.MustValue(name, "auth")
|
||||
if authUrl == "" {
|
||||
return nil, errors.New("auth not found")
|
||||
}
|
||||
c := &swift.Connection{
|
||||
UserName: *userName,
|
||||
ApiKey: *apiKey,
|
||||
AuthUrl: *authUrl,
|
||||
UserName: userName,
|
||||
ApiKey: apiKey,
|
||||
AuthUrl: authUrl,
|
||||
}
|
||||
err := c.Authenticate()
|
||||
if err != nil {
|
||||
|
|
@ -95,7 +118,7 @@ func swiftConnection() (*swift.Connection, error) {
|
|||
}
|
||||
|
||||
// NewFs contstructs an FsSwift from the path, container:path
|
||||
func NewFs(path string) (fs.Fs, error) {
|
||||
func NewFs(name, path string) (fs.Fs, error) {
|
||||
container, directory, err := parsePath(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
@ -103,7 +126,7 @@ func NewFs(path string) (fs.Fs, error) {
|
|||
if directory != "" {
|
||||
return nil, fmt.Errorf("Directories not supported yet in %q", path)
|
||||
}
|
||||
c, err := swiftConnection()
|
||||
c, err := swiftConnection(name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue