community
/
rclone
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fs: change Config callback into state based callback #3455 

This is a very large change which turns the post Config function in
backends into a state based call and response system so that
alternative user interfaces can be added.

The existing config logic has been converted, but it is quite
complicated and folloup commits will likely be needed to fix it!

Follow up commits will add a command line and API based way of using
this configuration system.
This commit is contained in:
Nick Craig-Wood 2021-04-29 09:28:18 +01:00
parent 6f2e525821
commit 94dbfa4ea6
25 changed files with 1370 additions and 940 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
backend/amazonclouddrive/amazonclouddrive.go
View File

@ -69,12 +69,10 @@ func init() {
Prefix: "acd",
Description: "Amazon Drive",
NewFs: NewFs,
Config: func(ctx context.Context, name string, m configmap.Mapper) error {
err := oauthutil.Config(ctx, "amazon cloud drive", name, m, acdConfig, nil)
if err != nil {
return errors.Wrap(err, "failed to configure token")
}
return nil
Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
return oauthutil.ConfigOut("", &oauthutil.Options{
OAuth2Config: acdConfig,
})
},
Options: append(oauthutil.SharedOptions, []fs.Option{{
Name: "checkpoint",

13
backend/box/box.go
View File

@ -83,7 +83,7 @@ func init() {
Name: "box",
Description: "Box",
NewFs: NewFs,
Config: func(ctx context.Context, name string, m configmap.Mapper) error {
Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
jsonFile, ok := m.Get("box_config_file")
boxSubType, boxSubTypeOk := m.Get("box_sub_type")
boxAccessToken, boxAccessTokenOk := m.Get("access_token")
@ -92,16 +92,15 @@ func init() {
if ok && boxSubTypeOk && jsonFile != "" && boxSubType != "" {
err = refreshJWTToken(ctx, jsonFile, boxSubType, name, m)
if err != nil {
return errors.Wrap(err, "failed to configure token with jwt authentication")
return nil, errors.Wrap(err, "failed to configure token with jwt authentication")
}
// Else, if not using an access token, use oauth2
} else if boxAccessToken == "" || !boxAccessTokenOk {
err = oauthutil.Config(ctx, "box", name, m, oauthConfig, nil)
if err != nil {
return errors.Wrap(err, "failed to configure token with oauth authentication")
}
return oauthutil.ConfigOut("", &oauthutil.Options{
OAuth2Config: oauthConfig,
})
}
return nil
return nil, nil
},
Options: append(oauthutil.SharedOptions, []fs.Option{{
Name: "root_folder_id",

110
backend/drive/drive.go
View File

@ -182,32 +182,64 @@ func init() {
Description: "Google Drive",
NewFs: NewFs,
CommandHelp: commandHelp,
Config: func(ctx context.Context, name string, m configmap.Mapper) error {
Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
// Parse config into Options struct
opt := new(Options)
err := configstruct.Set(m, opt)
if err != nil {
return errors.Wrap(err, "couldn't parse config into struct")
return nil, errors.Wrap(err, "couldn't parse config into struct")
}
// Fill in the scopes
driveConfig.Scopes = driveScopes(opt.Scope)
// Set the root_folder_id if using drive.appfolder
if driveScopesContainsAppFolder(driveConfig.Scopes) {
m.Set("root_folder_id", "appDataFolder")
}
switch config.State {
case "":
// Fill in the scopes
driveConfig.Scopes = driveScopes(opt.Scope)
if opt.ServiceAccountFile == "" && opt.ServiceAccountCredentials == "" {
err = oauthutil.Config(ctx, "drive", name, m, driveConfig, nil)
if err != nil {
return errors.Wrap(err, "failed to configure token")
// Set the root_folder_id if using drive.appfolder
if driveScopesContainsAppFolder(driveConfig.Scopes) {
m.Set("root_folder_id", "appDataFolder")
}
if opt.ServiceAccountFile == "" && opt.ServiceAccountCredentials == "" {
return oauthutil.ConfigOut("teamdrive", &oauthutil.Options{
OAuth2Config: driveConfig,
})
}
return fs.ConfigGoto("teamdrive")
case "teamdrive":
if opt.TeamDriveID == "" {
return fs.ConfigConfirm("teamdrive_ok", false, "Configure this as a Shared Drive (Team Drive)?\n")
}
return fs.ConfigConfirm("teamdrive_ok", false, fmt.Sprintf("Change current Shared Drive (Team Drive) ID %q?\n", opt.TeamDriveID))
case "teamdrive_ok":
if config.Result == "false" {
m.Set("team_drive", "")
return nil, nil
}
f, err := newFs(ctx, name, "", m)
if err != nil {
return nil, errors.Wrap(err, "failed to make Fs to list Shared Drives")
}
teamDrives, err := f.listTeamDrives(ctx)
if err != nil {
return nil, err
}
if len(teamDrives) == 0 {
return fs.ConfigError("", "No Shared Drives found in your account")
}
return fs.ConfigChoose("teamdrive_final", "Shared Drive", len(teamDrives), func(i int) (string, string) {
teamDrive := teamDrives[i]
return teamDrive.Id, teamDrive.Name
})
case "teamdrive_final":
driveID := config.Result
m.Set("team_drive", driveID)
m.Set("root_folder_id", "")
opt.TeamDriveID = driveID
opt.RootFolderID = ""
return nil, nil
}
err = configTeamDrive(ctx, opt, m, name)
if err != nil {
return errors.Wrap(err, "failed to configure Shared Drive")
}
return nil
return nil, fmt.Errorf("unknown state %q", config.State)
},
Options: append(driveOAuthOptions(), []fs.Option{{
Name: "scope",
@ -948,48 +980,6 @@ func parseExtensions(extensionsIn ...string) (extensions, mimeTypes []string, er
return
}
// Figure out if the user wants to use a team drive
func configTeamDrive(ctx context.Context, opt *Options, m configmap.Mapper, name string) error {
ci := fs.GetConfig(ctx)
// Stop if we are running non-interactive config
if ci.AutoConfirm {
return nil
}
if opt.TeamDriveID == "" {
fmt.Printf("Configure this as a Shared Drive (Team Drive)?\n")
} else {
fmt.Printf("Change current Shared Drive (Team Drive) ID %q?\n", opt.TeamDriveID)
}
if !config.Confirm(false) {
return nil
}
f, err := newFs(ctx, name, "", m)
if err != nil {
return errors.Wrap(err, "failed to make Fs to list Shared Drives")
}
fmt.Printf("Fetching Shared Drive list...\n")
teamDrives, err := f.listTeamDrives(ctx)
if err != nil {
return err
}
if len(teamDrives) == 0 {
fmt.Printf("No Shared Drives found in your account")
return nil
}
var driveIDs, driveNames []string
for _, teamDrive := range teamDrives {
driveIDs = append(driveIDs, teamDrive.Id)
driveNames = append(driveNames, teamDrive.Name)
}
driveID := config.Choose("Enter a Shared Drive ID", driveIDs, driveNames, true)
m.Set("team_drive", driveID)
m.Set("root_folder_id", "")
opt.TeamDriveID = driveID
opt.RootFolderID = ""
return nil
}
// getClient makes an http client according to the options
func getClient(ctx context.Context, opt *Options) *http.Client {
t := fshttp.NewTransportCustom(ctx, func(t *http.Transport) {
@ -1168,7 +1158,7 @@ func NewFs(ctx context.Context, name, path string, m configmap.Mapper) (fs.Fs, e
}
}
f.rootFolderID = rootID
fs.Debugf(f, "root_folder_id = %q - save this in the config to speed up startup", rootID)
fs.Debugf(f, "'root_folder_id = %s' - save this in the config to speed up startup", rootID)
}
f.dirCache = dircache.New(f.root, f.rootFolderID, f)

14
backend/dropbox/dropbox.go
View File

@ -143,18 +143,14 @@ func init() {
Name: "dropbox",
Description: "Dropbox",
NewFs: NewFs,
Config: func(ctx context.Context, name string, m configmap.Mapper) error {
opt := oauthutil.Options{
NoOffline: true,
Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
return oauthutil.ConfigOut("", &oauthutil.Options{
OAuth2Config: getOauthConfig(m),
NoOffline: true,
OAuth2Opts: []oauth2.AuthCodeOption{
oauth2.SetAuthURLParam("token_access_type", "offline"),
},
}
err := oauthutil.Config(ctx, "dropbox", name, m, getOauthConfig(m), &opt)
if err != nil {
return errors.Wrap(err, "failed to configure token")
}
return nil
})
},
Options: append(oauthutil.SharedOptions, []fs.Option{{
Name: "chunk_size",

12
backend/googlecloudstorage/googlecloudstorage.go
View File

@ -75,18 +75,16 @@ func init() {
Prefix: "gcs",
Description: "Google Cloud Storage (this is not Google Drive)",
NewFs: NewFs,
Config: func(ctx context.Context, name string, m configmap.Mapper) error {
Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
saFile, _ := m.Get("service_account_file")
saCreds, _ := m.Get("service_account_credentials")
anonymous, _ := m.Get("anonymous")
if saFile != "" || saCreds != "" || anonymous == "true" {
return nil
return nil, nil
}
err := oauthutil.Config(ctx, "google cloud storage", name, m, storageConfig, nil)
if err != nil {
return errors.Wrap(err, "failed to configure token")
}
return nil
return oauthutil.ConfigOut("", &oauthutil.Options{
OAuth2Config: storageConfig,
})
},
Options: append(oauthutil.SharedOptions, []fs.Option{{
Name: "project_number",

46
backend/googlephotos/googlephotos.go
View File

@ -77,36 +77,36 @@ func init() {
Prefix: "gphotos",
Description: "Google Photos",
NewFs: NewFs,
Config: func(ctx context.Context, name string, m configmap.Mapper) error {
Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
// Parse config into Options struct
opt := new(Options)
err := configstruct.Set(m, opt)
if err != nil {
return errors.Wrap(err, "couldn't parse config into struct")
return nil, errors.Wrap(err, "couldn't parse config into struct")
}
// Fill in the scopes
if opt.ReadOnly {
oauthConfig.Scopes[0] = scopeReadOnly
} else {
oauthConfig.Scopes[0] = scopeReadWrite
switch config.State {
case "":
// Fill in the scopes
if opt.ReadOnly {
oauthConfig.Scopes[0] = scopeReadOnly
} else {
oauthConfig.Scopes[0] = scopeReadWrite
}
return oauthutil.ConfigOut("warning", &oauthutil.Options{
OAuth2Config: oauthConfig,
})
case "warning":
// Warn the user as required by google photos integration
return fs.ConfigConfirm("warning_done", true, `Warning
IMPORTANT: All media items uploaded to Google Photos with rclone
are stored in full resolution at original quality. These uploads
will count towards storage in your Google Account.`)
case "warning_done":
return nil, nil
}
// Do the oauth
err = oauthutil.Config(ctx, "google photos", name, m, oauthConfig, nil)
if err != nil {
return errors.Wrap(err, "failed to configure token")
}
// Warn the user
fmt.Print(`
*** IMPORTANT: All media items uploaded to Google Photos with rclone
*** are stored in full resolution at original quality. These uploads
*** will count towards storage in your Google Account.
`)
return nil
return nil, fmt.Errorf("unknown state %q", config.State)
},
Options: append(oauthutil.SharedOptions, []fs.Option{{
Name: "read_only",

10
backend/hubic/hubic.go
View File

@ -55,12 +55,10 @@ func init() {
Name: "hubic",
Description: "Hubic",
NewFs: NewFs,
Config: func(ctx context.Context, name string, m configmap.Mapper) error {
err := oauthutil.Config(ctx, "hubic", name, m, oauthConfig, nil)
if err != nil {
return errors.Wrap(err, "failed to configure token")
}
return nil
Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
return oauthutil.ConfigOut("", &oauthutil.Options{
OAuth2Config: oauthConfig,
})
},
Options: append(oauthutil.SharedOptions, swift.SharedOptions...),
})

452
backend/jottacloud/jottacloud.go
View File

@ -55,6 +55,7 @@ const (
configTokenURL = "tokenURL"
configClientID = "client_id"
configClientSecret = "client_secret"
configUsername = "username"
configVersion = 1
v1tokenURL = "https://api.jottacloud.com/auth/v1/token"
@ -86,44 +87,7 @@ func init() {
Name: "jottacloud",
Description: "Jottacloud",
NewFs: NewFs,
Config: func(ctx context.Context, name string, m configmap.Mapper) error {
refresh := false
if version, ok := m.Get("configVersion"); ok {
ver, err := strconv.Atoi(version)
if err != nil {
return errors.Wrap(err, "failed to parse config version - corrupted config")
}
refresh = (ver != configVersion) && (ver != v1configVersion)
}
if refresh {
fmt.Printf("Config outdated - refreshing\n")
} else {
tokenString, ok := m.Get("token")
if ok && tokenString != "" {
fmt.Printf("Already have a token - refresh?\n")
if !config.Confirm(false) {
return nil
}
}
}
fmt.Printf("Choose authentication type:\n" +
"1: Standard authentication - use this if you're a normal Jottacloud user.\n" +
"2: Legacy authentication - this is only required for certain whitelabel versions of Jottacloud and not recommended for normal users.\n" +
"3: Telia Cloud authentication - use this if you are using Telia Cloud.\n")
switch config.ChooseNumber("Your choice", 1, 3) {
case 1:
return v2config(ctx, name, m)
case 2:
return v1config(ctx, name, m)
case 3:
return teliaCloudConfig(ctx, name, m)
default:
return errors.New("unknown config choice")
}
},
Config: Config,
Options: []fs.Option{{
Name: "md5_memory_limit",
Help: "Files bigger than this will be cached on disk to calculate the MD5 if required.",
@ -158,6 +122,181 @@ func init() {
})
}
// Config runs the backend configuration protocol
func Config(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
switch config.State {
case "":
return fs.ConfigChooseFixed("auth_type_done", `Authentication type`, []fs.OptionExample{{
Value: "standard",
Help: "Standard authentication - use this if you're a normal Jottacloud user.",
}, {
Value: "legacy",
Help: "Legacy authentication - this is only required for certain whitelabel versions of Jottacloud and not recommended for normal users.",
}, {
Value: "telia",
Help: "Telia Cloud authentication - use this if you are using Telia Cloud.",
}})
case "auth_type_done":
// Jump to next state according to config chosen
return fs.ConfigGoto(config.Result)
case "standard": // configure a jottacloud backend using the modern JottaCli token based authentication
m.Set("configVersion", fmt.Sprint(configVersion))
return fs.ConfigInput("standard_token", "Personal login token.\n\nGenerate here: https://www.jottacloud.com/web/secure")
case "standard_token":
loginToken := config.Result
m.Set(configClientID, "jottacli")
m.Set(configClientSecret, "")
srv := rest.NewClient(fshttp.NewClient(ctx))
token, err := doAuthV2(ctx, srv, loginToken, m)
if err != nil {
return nil, errors.Wrap(err, "failed to get oauth token")
}
err = oauthutil.PutToken(name, m, &token, true)
if err != nil {
return nil, errors.Wrap(err, "error while saving token")
}
return fs.ConfigGoto("choose_device")
case "legacy": // configure a jottacloud backend using legacy authentication
m.Set("configVersion", fmt.Sprint(v1configVersion))
return fs.ConfigConfirm("legacy_api", false, `Do you want to create a machine specific API key?
Rclone has it's own Jottacloud API KEY which works fine as long as one
only uses rclone on a single machine. When you want to use rclone with
this account on more than one machine it's recommended to create a
machine specific API key. These keys can NOT be shared between
machines.`)
case "legacy_api":
srv := rest.NewClient(fshttp.NewClient(ctx))
if config.Result == "true" {
deviceRegistration, err := registerDevice(ctx, srv)
if err != nil {
return nil, errors.Wrap(err, "failed to register device")
}
m.Set(configClientID, deviceRegistration.ClientID)
m.Set(configClientSecret, obscure.MustObscure(deviceRegistration.ClientSecret))
fs.Debugf(nil, "Got clientID %q and clientSecret %q", deviceRegistration.ClientID, deviceRegistration.ClientSecret)
}
return fs.ConfigInput("legacy_user", "Username")
case "legacy_username":
m.Set(configUsername, config.Result)
return fs.ConfigPassword("legacy_password", "Jottacloud password\n\n(this is only required during setup and will not be stored).")
case "legacy_password":
m.Set("password", config.Result)
m.Set("auth_code", "")
return fs.ConfigGoto("legacy_do_auth")
case "legacy_auth_code":
authCode := strings.Replace(config.Result, "-", "", -1) // remove any "-" contained in the code so we have a 6 digit number
m.Set("auth_code", authCode)
return fs.ConfigGoto("legacy_do_auth")
case "legacy_do_auth":
username, _ := m.Get(configUsername)
password, _ := m.Get("password")
authCode, _ := m.Get("auth_code")
srv := rest.NewClient(fshttp.NewClient(ctx))
clientID, ok := m.Get(configClientID)
if !ok {
clientID = v1ClientID
}
clientSecret, ok := m.Get(configClientSecret)
if !ok {
clientSecret = v1EncryptedClientSecret
}
// FIXME this is setting a global variable
oauthConfig.ClientID = clientID
oauthConfig.ClientSecret = obscure.MustReveal(clientSecret)
oauthConfig.Endpoint.AuthURL = v1tokenURL
oauthConfig.Endpoint.TokenURL = v1tokenURL
token, err := doAuthV1(ctx, srv, username, password, authCode)
if err == errAuthCodeRequired {
return fs.ConfigInput("legacy_auth_code", "Verification Code\nThis account uses 2 factor authentication you will receive a verification code via SMS.")
}
m.Set("password", "")
m.Set("auth_code", "")
if err != nil {
return nil, errors.Wrap(err, "failed to get oauth token")
}
err = oauthutil.PutToken(name, m, &token, true)
if err != nil {
return nil, errors.Wrap(err, "error while saving token")
}
return fs.ConfigGoto("choose_device")
case "telia": // telia cloud config
m.Set("configVersion", fmt.Sprint(configVersion))
m.Set(configClientID, teliaCloudClientID)
m.Set(configTokenURL, teliaCloudTokenURL)
return oauthutil.ConfigOut("choose_device", &oauthutil.Options{
OAuth2Config: &oauth2.Config{
Endpoint: oauth2.Endpoint{
AuthURL: teliaCloudAuthURL,
TokenURL: teliaCloudTokenURL,
},
ClientID: teliaCloudClientID,
Scopes: []string{"openid", "jotta-default", "offline_access"},
RedirectURL: oauthutil.RedirectLocalhostURL,
},
})
case "choose_device":
return fs.ConfigConfirm("choose_device_query", false, "Use a non standard device/mountpoint e.g. for accessing files uploaded using the official Jottacloud client?")
case "choose_device_query":
if config.Result != "true" {
m.Set(configDevice, "")
m.Set(configMountpoint, "")
return fs.ConfigGoto("end")
}
oAuthClient, _, err := getOAuthClient(ctx, name, m)
if err != nil {
return nil, err
}
srv := rest.NewClient(oAuthClient).SetRoot(rootURL)
apiSrv := rest.NewClient(oAuthClient).SetRoot(apiURL)
cust, err := getCustomerInfo(ctx, apiSrv)
if err != nil {
return nil, err
}
m.Set(configUsername, cust.Username)
acc, err := getDriveInfo(ctx, srv, cust.Username)
if err != nil {
return nil, err
}
return fs.ConfigChoose("choose_device_result", `Please select the device to use. Normally this will be Jotta`, len(acc.Devices), func(i int) (string, string) {
return acc.Devices[i].Name, ""
})
case "choose_device_result":
device := config.Result
m.Set(configDevice, device)
oAuthClient, _, err := getOAuthClient(ctx, name, m)
if err != nil {
return nil, err
}
srv := rest.NewClient(oAuthClient).SetRoot(rootURL)
username, _ := m.Get(configUsername)
dev, err := getDeviceInfo(ctx, srv, path.Join(username, device))
if err != nil {
return nil, err
}
return fs.ConfigChoose("choose_device_mountpoint", `Please select the mountpoint to use. Normally this will be Archive.`, len(dev.MountPoints), func(i int) (string, string) {
return dev.MountPoints[i].Name, ""
})
case "choose_device_mountpoint":
mountpoint := config.Result
m.Set(configMountpoint, mountpoint)
return fs.ConfigGoto("end")
case "end":
// All the config flows end up here in case we need to carry on with something
return nil, nil
}
return nil, fmt.Errorf("unknown state %q", config.State)
}
// Options defines the configuration for this backend
type Options struct {
Device string `config:"device"`
@ -243,111 +382,6 @@ func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, err
return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
}
func teliaCloudConfig(ctx context.Context, name string, m configmap.Mapper) error {
teliaCloudOauthConfig := &oauth2.Config{
Endpoint: oauth2.Endpoint{
AuthURL: teliaCloudAuthURL,
TokenURL: teliaCloudTokenURL,
},
ClientID: teliaCloudClientID,
Scopes: []string{"openid", "jotta-default", "offline_access"},
RedirectURL: oauthutil.RedirectLocalhostURL,
}
err := oauthutil.Config(ctx, "jottacloud", name, m, teliaCloudOauthConfig, nil)
if err != nil {
return errors.Wrap(err, "failed to configure token")
}
fmt.Printf("\nDo you want to use a non standard device/mountpoint e.g. for accessing files uploaded using the official Jottacloud client?\n\n")
if config.Confirm(false) {
oAuthClient, _, err := oauthutil.NewClient(ctx, name, m, teliaCloudOauthConfig)
if err != nil {
return errors.Wrap(err, "failed to load oAuthClient")
}
srv := rest.NewClient(oAuthClient).SetRoot(rootURL)
apiSrv := rest.NewClient(oAuthClient).SetRoot(apiURL)
device, mountpoint, err := setupMountpoint(ctx, srv, apiSrv)
if err != nil {
return errors.Wrap(err, "failed to setup mountpoint")
}
m.Set(configDevice, device)
m.Set(configMountpoint, mountpoint)
}
m.Set("configVersion", strconv.Itoa(configVersion))
m.Set(configClientID, teliaCloudClientID)
m.Set(configTokenURL, teliaCloudTokenURL)
return nil
}
// v1config configure a jottacloud backend using legacy authentication
func v1config(ctx context.Context, name string, m configmap.Mapper) error {
srv := rest.NewClient(fshttp.NewClient(ctx))
fmt.Printf("\nDo you want to create a machine specific API key?\n\nRclone has it's own Jottacloud API KEY which works fine as long as one only uses rclone on a single machine. When you want to use rclone with this account on more than one machine it's recommended to create a machine specific API key. These keys can NOT be shared between machines.\n\n")
if config.Confirm(false) {
deviceRegistration, err := registerDevice(ctx, srv)
if err != nil {
return errors.Wrap(err, "failed to register device")
}
m.Set(configClientID, deviceRegistration.ClientID)
m.Set(configClientSecret, obscure.MustObscure(deviceRegistration.ClientSecret))
fs.Debugf(nil, "Got clientID '%s' and clientSecret '%s'", deviceRegistration.ClientID, deviceRegistration.ClientSecret)
}
clientID, ok := m.Get(configClientID)
if !ok {
clientID = v1ClientID
}
clientSecret, ok := m.Get(configClientSecret)
if !ok {
clientSecret = v1EncryptedClientSecret
}
oauthConfig.ClientID = clientID
oauthConfig.ClientSecret = obscure.MustReveal(clientSecret)
oauthConfig.Endpoint.AuthURL = v1tokenURL
oauthConfig.Endpoint.TokenURL = v1tokenURL
fmt.Printf("Username> ")
username := config.ReadLine()
password := config.GetPassword("Your Jottacloud password is only required during setup and will not be stored.")
token, err := doAuthV1(ctx, srv, username, password)
if err != nil {
return errors.Wrap(err, "failed to get oauth token")
}
err = oauthutil.PutToken(name, m, &token, true)
if err != nil {
return errors.Wrap(err, "error while saving token")
}
fmt.Printf("\nDo you want to use a non standard device/mountpoint e.g. for accessing files uploaded using the official Jottacloud client?\n\n")
if config.Confirm(false) {
oAuthClient, _, err := oauthutil.NewClient(ctx, name, m, oauthConfig)
if err != nil {
return errors.Wrap(err, "failed to load oAuthClient")
}
srv = rest.NewClient(oAuthClient).SetRoot(rootURL)
apiSrv := rest.NewClient(oAuthClient).SetRoot(apiURL)
device, mountpoint, err := setupMountpoint(ctx, srv, apiSrv)
if err != nil {
return errors.Wrap(err, "failed to setup mountpoint")
}
m.Set(configDevice, device)
m.Set(configMountpoint, mountpoint)
}
m.Set("configVersion", strconv.Itoa(v1configVersion))
return nil
}
// registerDevice register a new device for use with the jottacloud API
func registerDevice(ctx context.Context, srv *rest.Client) (reg *api.DeviceRegistrationResponse, err error) {
// random generator to generate random device names
@ -377,8 +411,13 @@ func registerDevice(ctx context.Context, srv *rest.Client) (reg *api.DeviceRegis
return deviceRegistration, err
}
var errAuthCodeRequired = errors.New("auth code required")
// doAuthV1 runs the actual token request for V1 authentication
func doAuthV1(ctx context.Context, srv *rest.Client, username, password string) (token oauth2.Token, err error) {
//
// Call this first with blank authCode. If errAuthCodeRequired is
// returned then call it again with an authCode
func doAuthV1(ctx context.Context, srv *rest.Client, username, password, authCode string) (token oauth2.Token, err error) {
// prepare out token request with username and password
values := url.Values{}
values.Set("grant_type", "PASSWORD")
@ -392,22 +431,19 @@ func doAuthV1(ctx context.Context, srv *rest.Client, username, password string)
ContentType: "application/x-www-form-urlencoded",
Parameters: values,
}
if authCode != "" {
opts.ExtraHeaders = make(map[string]string)
opts.ExtraHeaders["X-Jottacloud-Otp"] = authCode
}
// do the first request
var jsonToken api.TokenJSON
resp, err := srv.CallJSON(ctx, &opts, nil, &jsonToken)
if err != nil {
if err != nil && authCode == "" {
// if 2fa is enabled the first request is expected to fail. We will do another request with the 2fa code as an additional http header
if resp != nil {
if resp.Header.Get("X-JottaCloud-OTP") == "required; SMS" {
fmt.Printf("This account uses 2 factor authentication you will receive a verification code via SMS.\n")
fmt.Printf("Enter verification code> ")
authCode := config.ReadLine()
authCode = strings.Replace(authCode, "-", "", -1) // remove any "-" contained in the code so we have a 6 digit number
opts.ExtraHeaders = make(map[string]string)
opts.ExtraHeaders["X-Jottacloud-Otp"] = authCode
_, err = srv.CallJSON(ctx, &opts, nil, &jsonToken)
return token, errAuthCodeRequired
}
}
}
@ -419,47 +455,6 @@ func doAuthV1(ctx context.Context, srv *rest.Client, username, password string)
return token, err
}
// v2config configure a jottacloud backend using the modern JottaCli token based authentication
func v2config(ctx context.Context, name string, m configmap.Mapper) error {
srv := rest.NewClient(fshttp.NewClient(ctx))
fmt.Printf("Generate a personal login token here: https://www.jottacloud.com/web/secure\n")
fmt.Printf("Login Token> ")
loginToken := config.ReadLine()
m.Set(configClientID, "jottacli")
m.Set(configClientSecret, "")
token, err := doAuthV2(ctx, srv, loginToken, m)
if err != nil {
return errors.Wrap(err, "failed to get oauth token")
}
err = oauthutil.PutToken(name, m, &token, true)
if err != nil {
return errors.Wrap(err, "error while saving token")
}
fmt.Printf("\nDo you want to use a non standard device/mountpoint e.g. for accessing files uploaded using the official Jottacloud client?\n\n")
if config.Confirm(false) {
oAuthClient, _, err := oauthutil.NewClient(ctx, name, m, oauthConfig)
if err != nil {
return errors.Wrap(err, "failed to load oAuthClient")
}
srv = rest.NewClient(oAuthClient).SetRoot(rootURL)
apiSrv := rest.NewClient(oAuthClient).SetRoot(apiURL)
device, mountpoint, err := setupMountpoint(ctx, srv, apiSrv)
if err != nil {
return errors.Wrap(err, "failed to setup mountpoint")
}
m.Set(configDevice, device)
m.Set(configMountpoint, mountpoint)
}
m.Set("configVersion", strconv.Itoa(configVersion))
return nil
}
// doAuthV2 runs the actual token request for V2 authentication
func doAuthV2(ctx context.Context, srv *rest.Client, loginTokenBase64 string, m configmap.Mapper) (token oauth2.Token, err error) {
loginTokenBytes, err := base64.RawURLEncoding.DecodeString(loginTokenBase64)
@ -520,41 +515,6 @@ func doAuthV2(ctx context.Context, srv *rest.Client, loginTokenBase64 string, m
return token, err
}
// setupMountpoint sets up a custom device and mountpoint if desired by the user
func setupMountpoint(ctx context.Context, srv *rest.Client, apiSrv *rest.Client) (device, mountpoint string, err error) {
cust, err := getCustomerInfo(ctx, apiSrv)
if err != nil {
return "", "", err
}
acc, err := getDriveInfo(ctx, srv, cust.Username)
if err != nil {
return "", "", err
}
var deviceNames []string
for i := range acc.Devices {
deviceNames = append(deviceNames, acc.Devices[i].Name)
}
fmt.Printf("Please select the device to use. Normally this will be Jotta\n")
device = config.Choose("Devices", deviceNames, nil, false)
dev, err := getDeviceInfo(ctx, srv, path.Join(cust.Username, device))
if err != nil {
return "", "", err
}
if len(dev.MountPoints) == 0 {
return "", "", errors.New("no mountpoints for selected device")
}
var mountpointNames []string
for i := range dev.MountPoints {
mountpointNames = append(mountpointNames, dev.MountPoints[i].Name)
}
fmt.Printf("Please select the mountpoint to user. Normally this will be Archive\n")
mountpoint = config.Choose("Mountpoints", mountpointNames, nil, false)
return device, mountpoint, err
}
// getCustomerInfo queries general information about the account
func getCustomerInfo(ctx context.Context, srv *rest.Client) (info *api.CustomerInfo, err error) {
opts := rest.Opts{
@ -695,27 +655,19 @@ func grantTypeFilter(req *http.Request) {
}
}
// NewFs constructs an Fs from the path, container:path
func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
// Parse config into Options struct
opt := new(Options)
err := configstruct.Set(m, opt)
if err != nil {
return nil, err
}
func getOAuthClient(ctx context.Context, name string, m configmap.Mapper) (oAuthClient *http.Client, ts *oauthutil.TokenSource, err error) {
// Check config version
var ver int
version, ok := m.Get("configVersion")
if ok {
ver, err = strconv.Atoi(version)
if err != nil {
return nil, errors.New("Failed to parse config version")
return nil, nil, errors.New("Failed to parse config version")
}
ok = (ver == configVersion) || (ver == v1configVersion)
}
if !ok {
return nil, errors.New("Outdated config - please reconfigure this backend")
return nil, nil, errors.New("Outdated config - please reconfigure this backend")
}
baseClient := fshttp.NewClient(ctx)
@ -754,9 +706,25 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
}
// Create OAuth Client
oAuthClient, ts, err := oauthutil.NewClientWithBaseClient(ctx, name, m, oauthConfig, baseClient)
oAuthClient, ts, err = oauthutil.NewClientWithBaseClient(ctx, name, m, oauthConfig, baseClient)
if err != nil {
return nil, errors.Wrap(err, "Failed to configure Jottacloud oauth client")
return nil, nil, errors.Wrap(err, "Failed to configure Jottacloud oauth client")
}
return oAuthClient, ts, nil
}
// NewFs constructs an Fs from the path, container:path
func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
// Parse config into Options struct
opt := new(Options)
err := configstruct.Set(m, opt)
if err != nil {
return nil, err
}
oAuthClient, ts, err := getOAuthClient(ctx, name, m)
if err != nil {
return nil, err
}
rootIsDir := strings.HasSuffix(root, "/")

460
backend/onedrive/onedrive.go
View File

@ -98,208 +98,7 @@ func init() {
Name: "onedrive",
Description: "Microsoft OneDrive",
NewFs: NewFs,
Config: func(ctx context.Context, name string, m configmap.Mapper) error {
region, _ := m.Get("region")
graphURL := graphAPIEndpoint[region] + "/v1.0"
oauthConfig.Endpoint = oauth2.Endpoint{
AuthURL: authEndpoint[region] + authPath,
TokenURL: authEndpoint[region] + tokenPath,
}
ci := fs.GetConfig(ctx)
err := oauthutil.Config(ctx, "onedrive", name, m, oauthConfig, nil)
if err != nil {
return errors.Wrap(err, "failed to configure token")
}
// Stop if we are running non-interactive config
if ci.AutoConfirm {
return nil
}
type driveResource struct {
DriveID string `json:"id"`
DriveName string `json:"name"`
DriveType string `json:"driveType"`
}
type drivesResponse struct {
Drives []driveResource `json:"value"`
}
type siteResource struct {
SiteID string `json:"id"`
SiteName string `json:"displayName"`
SiteURL string `json:"webUrl"`
}
type siteResponse struct {
Sites []siteResource `json:"value"`
}
oAuthClient, _, err := oauthutil.NewClient(ctx, name, m, oauthConfig)
if err != nil {
return errors.Wrap(err, "failed to configure OneDrive")
}
srv := rest.NewClient(oAuthClient)
var opts rest.Opts
var finalDriveID string
var siteID string
var relativePath string
switch config.Choose("Your choice",
[]string{"onedrive", "sharepoint", "url", "search", "driveid", "siteid", "path"},
[]string{
"OneDrive Personal or Business",
"Root Sharepoint site",
"Sharepoint site name or URL (e.g. mysite or https://contoso.sharepoint.com/sites/mysite)",
"Search for a Sharepoint site",
"Type in driveID (advanced)",
"Type in SiteID (advanced)",
"Sharepoint server-relative path (advanced, e.g. /teams/hr)",
},
false) {
case "onedrive":
opts = rest.Opts{
Method: "GET",
RootURL: graphURL,
Path: "/me/drives",
}
case "sharepoint":
opts = rest.Opts{
Method: "GET",
RootURL: graphURL,
Path: "/sites/root/drives",
}
case "driveid":
fmt.Printf("Paste your Drive ID here> ")
finalDriveID = config.ReadLine()
case "siteid":
fmt.Printf("Paste your Site ID here> ")
siteID = config.ReadLine()
case "url":
fmt.Println("Example: \"https://contoso.sharepoint.com/sites/mysite\" or \"mysite\"")
fmt.Printf("Paste your Site URL here> ")
siteURL := config.ReadLine()
re := regexp.MustCompile(`https://.*\.sharepoint.com/sites/(.*)`)
match := re.FindStringSubmatch(siteURL)
if len(match) == 2 {
relativePath = "/sites/" + match[1]
} else {
relativePath = "/sites/" + siteURL
}
case "path":
fmt.Printf("Enter server-relative URL here> ")
relativePath = config.ReadLine()
case "search":
fmt.Printf("What to search for> ")
searchTerm := config.ReadLine()
opts = rest.Opts{
Method: "GET",
RootURL: graphURL,
Path: "/sites?search=" + searchTerm,
}
sites := siteResponse{}
_, err := srv.CallJSON(ctx, &opts, nil, &sites)
if err != nil {
return errors.Wrap(err, "failed to query available sites")
}
if len(sites.Sites) == 0 {
return errors.Errorf("search for %q returned no results", searchTerm)
}
fmt.Printf("Found %d sites, please select the one you want to use:\n", len(sites.Sites))
for index, site := range sites.Sites {
fmt.Printf("%d: %s (%s) id=%s\n", index, site.SiteName, site.SiteURL, site.SiteID)
}
siteID = sites.Sites[config.ChooseNumber("Chose drive to use:", 0, len(sites.Sites)-1)].SiteID
}
// if we use server-relative URL for finding the drive
if relativePath != "" {
opts = rest.Opts{
Method: "GET",
RootURL: graphURL,
Path: "/sites/root:" + relativePath,
}
site := siteResource{}
_, err := srv.CallJSON(ctx, &opts, nil, &site)
if err != nil {
return errors.Wrap(err, "failed to query available site by relative path")
}
siteID = site.SiteID
}
// if we have a siteID we need to ask for the drives
if siteID != "" {
opts = rest.Opts{
Method: "GET",
RootURL: graphURL,
Path: "/sites/" + siteID + "/drives",
}
}
// We don't have the final ID yet?
// query Microsoft Graph
if finalDriveID == "" {
drives := drivesResponse{}
_, err := srv.CallJSON(ctx, &opts, nil, &drives)
if err != nil {
return errors.Wrap(err, "failed to query available drives")
}
// Also call /me/drive as sometimes /me/drives doesn't return it #4068
if opts.Path == "/me/drives" {
opts.Path = "/me/drive"
meDrive := driveResource{}
_, err := srv.CallJSON(ctx, &opts, nil, &meDrive)
if err != nil {
return errors.Wrap(err, "failed to query available drives")
}
found := false
for _, drive := range drives.Drives {
if drive.DriveID == meDrive.DriveID {
found = true
break
}
}
// add the me drive if not found already
if !found {
fs.Debugf(nil, "Adding %v to drives list from /me/drive", meDrive)
drives.Drives = append(drives.Drives, meDrive)
}
}
if len(drives.Drives) == 0 {
return errors.New("no drives found")
}
fmt.Printf("Found %d drives, please select the one you want to use:\n", len(drives.Drives))
for index, drive := range drives.Drives {
fmt.Printf("%d: %s (%s) id=%s\n", index, drive.DriveName, drive.DriveType, drive.DriveID)
}
finalDriveID = drives.Drives[config.ChooseNumber("Chose drive to use:", 0, len(drives.Drives)-1)].DriveID
}
// Test the driveID and get drive type
opts = rest.Opts{
Method: "GET",
RootURL: graphURL,
Path: "/drives/" + finalDriveID + "/root"}
var rootItem api.Item
_, err = srv.CallJSON(ctx, &opts, nil, &rootItem)
if err != nil {
return errors.Wrapf(err, "failed to query root for drive %s", finalDriveID)
}
fmt.Printf("Found drive '%s' of type '%s', URL: %s\nIs that okay?\n", rootItem.Name, rootItem.ParentReference.DriveType, rootItem.WebURL)
// This does not work, YET :)
if !config.ConfirmWithConfig(ctx, m, "config_drive_ok", true) {
return errors.New("cancelled by user")
}
m.Set(configDriveID, finalDriveID)
m.Set(configDriveType, rootItem.ParentReference.DriveType)
return nil
},
Config: Config,
Options: append(oauthutil.SharedOptions, []fs.Option{{
Name: "region",
Help: "Choose national cloud region for OneDrive.",
@ -462,6 +261,263 @@ At the time of writing this only works with OneDrive personal paid accounts.
})
}
type driveResource struct {
DriveID string `json:"id"`
DriveName string `json:"name"`
DriveType string `json:"driveType"`
}
type drivesResponse struct {
Drives []driveResource `json:"value"`
}
type siteResource struct {
SiteID string `json:"id"`
SiteName string `json:"displayName"`
SiteURL string `json:"webUrl"`
}
type siteResponse struct {
Sites []siteResource `json:"value"`
}
// Get the region and graphURL from the config
func getRegionURL(m configmap.Mapper) (region, graphURL string) {
region, _ = m.Get("region")
graphURL = graphAPIEndpoint[region] + "/v1.0"
return region, graphURL
}
// Config for chooseDrive
type chooseDriveOpt struct {
opts rest.Opts
finalDriveID string
siteID string
relativePath string
}
// chooseDrive returns a query to choose which drive the user is interested in
func chooseDrive(ctx context.Context, name string, m configmap.Mapper, srv *rest.Client, opt chooseDriveOpt) (*fs.ConfigOut, error) {
_, graphURL := getRegionURL(m)
// if we use server-relative URL for finding the drive
if opt.relativePath != "" {
opt.opts = rest.Opts{
Method: "GET",
RootURL: graphURL,
Path: "/sites/root:" + opt.relativePath,
}
site := siteResource{}
_, err := srv.CallJSON(ctx, &opt.opts, nil, &site)
if err != nil {
return fs.ConfigError("choose_type", fmt.Sprintf("Failed to query available site by relative path: %v", err))
}
opt.siteID = site.SiteID
}
// if we have a siteID we need to ask for the drives
if opt.siteID != "" {
opt.opts = rest.Opts{
Method: "GET",
RootURL: graphURL,
Path: "/sites/" + opt.siteID + "/drives",
}
}
drives := drivesResponse{}
// We don't have the final ID yet?
// query Microsoft Graph
if opt.finalDriveID == "" {
_, err := srv.CallJSON(ctx, &opt.opts, nil, &drives)
if err != nil {
return fs.ConfigError("choose_type", fmt.Sprintf("Failed to query available drives: %v", err))
}
// Also call /me/drive as sometimes /me/drives doesn't return it #4068
if opt.opts.Path == "/me/drives" {
opt.opts.Path = "/me/drive"
meDrive := driveResource{}
_, err := srv.CallJSON(ctx, &opt.opts, nil, &meDrive)
if err != nil {
return fs.ConfigError("choose_type", fmt.Sprintf("Failed to query available drives: %v", err))
}
found := false
for _, drive := range drives.Drives {
if drive.DriveID == meDrive.DriveID {
found = true
break
}
}
// add the me drive if not found already
if !found {
fs.Debugf(nil, "Adding %v to drives list from /me/drive", meDrive)
drives.Drives = append(drives.Drives, meDrive)
}
}
} else {
drives.Drives = append(drives.Drives, driveResource{
DriveID: opt.finalDriveID,
DriveName: "Chosen Drive ID",
DriveType: "drive",
})
}
if len(drives.Drives) == 0 {
return fs.ConfigError("choose_type", "No drives found")
}
return fs.ConfigChoose("driveid_final", "Select drive you want to use", len(drives.Drives), func(i int) (string, string) {
drive := drives.Drives[i]
return drive.DriveID, fmt.Sprintf("%s (%s)", drive.DriveName, drive.DriveType)
})
}
// Config the backend
func Config(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
oAuthClient, _, err := oauthutil.NewClient(ctx, name, m, oauthConfig)
if err != nil {
return nil, errors.Wrap(err, "failed to configure OneDrive")
}
srv := rest.NewClient(oAuthClient)
region, graphURL := getRegionURL(m)
switch config.State {
case "":
oauthConfig.Endpoint = oauth2.Endpoint{
AuthURL: authEndpoint[region] + authPath,
TokenURL: authEndpoint[region] + tokenPath,
}
return oauthutil.ConfigOut("choose_type", &oauthutil.Options{
OAuth2Config: oauthConfig,
})
case "choose_type":
return fs.ConfigChooseFixed("choose_type_done", "Type of connection", []fs.OptionExample{{
Value: "onedrive",
Help: "OneDrive Personal or Business",
}, {
Value: "sharepoint",
Help: "Root Sharepoint site",
}, {
Value: "url",
Help: "Sharepoint site name or URL (e.g. mysite or https://contoso.sharepoint.com/sites/mysite)",
}, {
Value: "search",
Help: "Search for a Sharepoint site",
}, {
Value: "driveid",
Help: "Type in driveID (advanced)",
}, {
Value: "siteid",
Help: "Type in SiteID (advanced)",
}, {
Value: "path",
Help: "Sharepoint server-relative path (advanced, e.g. /teams/hr)",
}})
case "choose_type_done":
// Jump to next state according to config chosen
return fs.ConfigGoto(config.Result)
case "onedrive":
return chooseDrive(ctx, name, m, srv, chooseDriveOpt{
opts: rest.Opts{
Method: "GET",
RootURL: graphURL,
Path: "/me/drives",
},
})
case "sharepoint":
return chooseDrive(ctx, name, m, srv, chooseDriveOpt{
opts: rest.Opts{
Method: "GET",
RootURL: graphURL,
Path: "/sites/root/drives",
},
})
case "driveid":
return fs.ConfigInput("driveid_end", "Drive ID")
case "driveid_end":
return chooseDrive(ctx, name, m, srv, chooseDriveOpt{
finalDriveID: config.Result,
})
case "siteid":
return fs.ConfigInput("siteid_end", "Site ID")
case "siteid_end":
return chooseDrive(ctx, name, m, srv, chooseDriveOpt{
siteID: config.Result,
})
case "url":
return fs.ConfigInput("url_end", `Site URL
Example: "https://contoso.sharepoint.com/sites/mysite" or "mysite"
`)
case "url_end":
siteURL := config.Result
re := regexp.MustCompile(`https://.*\.sharepoint.com/sites/(.*)`)
match := re.FindStringSubmatch(siteURL)
if len(match) == 2 {
return chooseDrive(ctx, name, m, srv, chooseDriveOpt{
relativePath: "/sites/" + match[1],
})
}
return chooseDrive(ctx, name, m, srv, chooseDriveOpt{
relativePath: "/sites/" + siteURL,
})
case "path":
return fs.ConfigInput("path_end", `Server-relative URL`)
case "path_end":
return chooseDrive(ctx, name, m, srv, chooseDriveOpt{
relativePath: config.Result,
})
case "search":
return fs.ConfigInput("search_end", `Search term`)
case "search_end":
searchTerm := config.Result
opts := rest.Opts{
Method: "GET",
RootURL: graphURL,
Path: "/sites?search=" + searchTerm,
}
sites := siteResponse{}
_, err := srv.CallJSON(ctx, &opts, nil, &sites)
if err != nil {
return fs.ConfigError("choose_type", fmt.Sprintf("Failed to query available sites: %v", err))
}
if len(sites.Sites) == 0 {
return fs.ConfigError("choose_type", fmt.Sprintf("search for %q returned no results", searchTerm))
}
return fs.ConfigChoose("search_sites", `Select the Site you want to use`, len(sites.Sites), func(i int) (string, string) {
site := sites.Sites[i]
return site.SiteID, fmt.Sprintf("%s (%s)", site.SiteName, site.SiteURL)
})
case "search_sites":
return chooseDrive(ctx, name, m, srv, chooseDriveOpt{
siteID: config.Result,
})
case "driveid_final":
finalDriveID := config.Result
// Test the driveID and get drive type
opts := rest.Opts{
Method: "GET",
RootURL: graphURL,
Path: "/drives/" + finalDriveID + "/root"}
var rootItem api.Item
_, err = srv.CallJSON(ctx, &opts, nil, &rootItem)
if err != nil {
return fs.ConfigError("choose_type", fmt.Sprintf("Failed to query root for drive %q: %v", finalDriveID, err))
}
m.Set(configDriveID, finalDriveID)
m.Set(configDriveType, rootItem.ParentReference.DriveType)
return fs.ConfigConfirm("driveid_final_end", true, fmt.Sprintf("Drive OK?\n\nFound drive %q of type %q\nURL: %s\n", rootItem.Name, rootItem.ParentReference.DriveType, rootItem.WebURL))
case "driveid_final_end":
if config.Result == "true" {
return nil, nil
}
return fs.ConfigGoto("choose_type")
}
return nil, fmt.Errorf("unknown state %q", config.State)
}
// Options defines the configuration for this backend
type Options struct {
Region string `config:"region"`

12
backend/pcloud/pcloud.go
View File

@ -71,7 +71,7 @@ func init() {
Name: "pcloud",
Description: "Pcloud",
NewFs: NewFs,
Config: func(ctx context.Context, name string, m configmap.Mapper) error {
Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
optc := new(Options)
err := configstruct.Set(m, optc)
if err != nil {
@ -93,15 +93,11 @@ func init() {
fs.Debugf(nil, "pcloud: got hostname %q", hostname)
return nil
}
opt := oauthutil.Options{
return oauthutil.ConfigOut("", &oauthutil.Options{
OAuth2Config: oauthConfig,
CheckAuth: checkAuth,
StateBlankOK: true, // pCloud seems to drop the state parameter now - see #4210
}
err = oauthutil.Config(ctx, "pcloud", name, m, oauthConfig, &opt)
if err != nil {
return errors.Wrap(err, "failed to configure token")
}
return nil
})
},
Options: append(oauthutil.SharedOptions, []fs.Option{{
Name: config.ConfigEncoding,

10
backend/premiumizeme/premiumizeme.go
View File

@ -77,12 +77,10 @@ func init() {
Name: "premiumizeme",
Description: "premiumize.me",
NewFs: NewFs,
Config: func(ctx context.Context, name string, m configmap.Mapper) error {
err := oauthutil.Config(ctx, "premiumizeme", name, m, oauthConfig, nil)
if err != nil {
return errors.Wrap(err, "failed to configure token")
}
return nil
Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
return oauthutil.ConfigOut("", &oauthutil.Options{
OAuth2Config: oauthConfig,
})
},
Options: []fs.Option{{
Name: "api_key",

15
backend/putio/putio.go
View File

@ -5,7 +5,6 @@ import (
"regexp"
"time"
"github.com/pkg/errors"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config"
"github.com/rclone/rclone/fs/config/configmap"
@ -60,15 +59,11 @@ func init() {
Name: "putio",
Description: "Put.io",
NewFs: NewFs,
Config: func(ctx context.Context, name string, m configmap.Mapper) error {
opt := oauthutil.Options{
NoOffline: true,
}
err := oauthutil.Config(ctx, "putio", name, m, putioConfig, &opt)
if err != nil {
return errors.Wrap(err, "failed to configure token")
}
return nil
Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
return oauthutil.ConfigOut("", &oauthutil.Options{
OAuth2Config: putioConfig,
NoOffline: true,
})
},
Options: []fs.Option{{
Name: config.ConfigEncoding,

93
backend/seafile/seafile.go
View File

@ -296,83 +296,86 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
}
// Config callback for 2FA
func Config(ctx context.Context, name string, m configmap.Mapper) error {
ci := fs.GetConfig(ctx)
func Config(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
serverURL, ok := m.Get(configURL)
if !ok || serverURL == "" {
// If there's no server URL, it means we're trying an operation at the backend level, like a "rclone authorize seafile"
return errors.New("operation not supported on this remote. If you need a 2FA code on your account, use the command: nrclone config reconnect <remote name>: ")
}
// Stop if we are running non-interactive config
if ci.AutoConfirm {
return nil
return nil, errors.New("operation not supported on this remote. If you need a 2FA code on your account, use the command: rclone config reconnect <remote name>: ")
}
u, err := url.Parse(serverURL)
if err != nil {
return errors.Errorf("invalid server URL %s", serverURL)
return nil, errors.Errorf("invalid server URL %s", serverURL)
}
is2faEnabled, _ := m.Get(config2FA)
if is2faEnabled != "true" {
return errors.New("two-factor authentication is not enabled on this account")
return nil, errors.New("two-factor authentication is not enabled on this account")
}
username, _ := m.Get(configUser)
if username == "" {
return errors.New("a username is required")
return nil, errors.New("a username is required")
}
password, _ := m.Get(configPassword)
if password != "" {
password, _ = obscure.Reveal(password)
}
// Just make sure we do have a password
for password == "" {
fmt.Print("Two-factor authentication: please enter your password (it won't be saved in the configuration)\npassword> ")
password = config.ReadPassword()
}
// Create rest client for getAuthorizationToken
url := u.String()
if !strings.HasPrefix(url, "/") {
url += "/"
}
srv := rest.NewClient(fshttp.NewClient(ctx)).SetRoot(url)
// We loop asking for a 2FA code
for {
code := ""
for code == "" {
fmt.Print("Two-factor authentication: please enter your 2FA code\n2fa code> ")
code = config.ReadLine()
switch config.State {
case "":
// Just make sure we do have a password
if password == "" {
return fs.ConfigPassword("", "Two-factor authentication: please enter your password (it won't be saved in the configuration)")
}
return fs.ConfigGoto("password")
case "password":
password = config.Result
if password == "" {
return fs.ConfigError("password", "Password can't be blank")
}
m.Set(configPassword, obscure.MustObscure(config.Result))
return fs.ConfigGoto("2fa")
case "2fa":
return fs.ConfigInput("2fa_do", "Two-factor authentication: please enter your 2FA code")
case "2fa_do":
code := config.Result
if code == "" {
return fs.ConfigError("2fa", "2FA codes can't be blank")
}
// Create rest client for getAuthorizationToken
url := u.String()
if !strings.HasPrefix(url, "/") {
url += "/"
}
srv := rest.NewClient(fshttp.NewClient(ctx)).SetRoot(url)
// We loop asking for a 2FA code
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
fmt.Println("Authenticating...")
token, err := getAuthorizationToken(ctx, srv, username, password, code)
if err != nil {
fmt.Printf("Authentication failed: %v\n", err)
tryAgain := strings.ToLower(config.ReadNonEmptyLine("Do you want to try again (y/n)?"))
if tryAgain != "y" && tryAgain != "yes" {
// The user is giving up, we're done here
break
}
return fs.ConfigConfirm("2fa_error", true, fmt.Sprintf("Authentication failed: %v\n\nTry Again?", err))
}
if token != "" {
fmt.Println("Success!")
// Let's save the token into the configuration
m.Set(configAuthToken, token)
// And delete any previous entry for password
m.Set(configPassword, "")
// And we're done here
break
if token == "" {
return fs.ConfigConfirm("2fa_error", true, "Authentication failed - no token returned.\n\nTry Again?")
}
// Let's save the token into the configuration
m.Set(configAuthToken, token)
// And delete any previous entry for password
m.Set(configPassword, "")
// And we're done here
return nil, nil
case "2fa_error":
if config.Result == "true" {
return fs.ConfigGoto("2fa")
}
return nil, errors.New("2fa authentication failed")
}
return nil
return nil, fmt.Errorf("unknown state %q", config.State)
}
// sets the AuthorizationToken up

14
backend/sharefile/sharefile.go
View File

@ -135,7 +135,7 @@ func init() {
Name: "sharefile",
Description: "Citrix Sharefile",
NewFs: NewFs,
Config: func(ctx context.Context, name string, m configmap.Mapper) error {
Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
oauthConfig := newOauthConfig("")
checkAuth := func(oauthConfig *oauth2.Config, auth *oauthutil.AuthResult) error {
if auth == nil || auth.Form == nil {
@ -151,14 +151,10 @@ func init() {
oauthConfig.Endpoint.TokenURL = endpoint + tokenPath
return nil
}
opt := oauthutil.Options{
CheckAuth: checkAuth,
}
err := oauthutil.Config(ctx, "sharefile", name, m, oauthConfig, &opt)
if err != nil {
return errors.Wrap(err, "failed to configure token")
}
return nil
return oauthutil.ConfigOut("", &oauthutil.Options{
OAuth2Config: oauthConfig,
CheckAuth: checkAuth,
})
},
Options: []fs.Option{{
Name: "upload_cutoff",

82
backend/sugarsync/sugarsync.go
View File

@ -75,51 +75,63 @@ func init() {
Name: "sugarsync",
Description: "Sugarsync",
NewFs: NewFs,
Config: func(ctx context.Context, name string, m configmap.Mapper) error {
Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
opt := new(Options)
err := configstruct.Set(m, opt)
if err != nil {
return errors.Wrap(err, "failed to read options")
return nil, errors.Wrap(err, "failed to read options")
}
if opt.RefreshToken != "" {
fmt.Printf("Already have a token - refresh?\n")
if !config.ConfirmWithConfig(ctx, m, "config_refresh_token", true) {
return nil
switch config.State {
case "":
if opt.RefreshToken == "" {
return fs.ConfigGoto("username")
}
}
fmt.Printf("Username (email address)> ")
username := config.ReadLine()
password := config.GetPassword("Your Sugarsync password is only required during setup and will not be stored.")
return fs.ConfigConfirm("refresh", true, "Already have a token - refresh?")
case "refresh":
if config.Result == "false" {
return nil, nil
}
return fs.ConfigGoto("username")
case "username":
return fs.ConfigInput("password", "username (email address)")
case "password":
m.Set("username", config.Result)
return fs.ConfigPassword("auth", "Your Sugarsync password.\n\nOnly required during setup and will not be stored.")
case "auth":
username, _ := m.Get("username")
m.Set("username", "")
password := config.Result
authRequest := api.AppAuthorization{
Username: username,
Password: password,
Application: withDefault(opt.AppID, appID),
AccessKeyID: withDefault(opt.AccessKeyID, accessKeyID),
PrivateAccessKey: withDefault(opt.PrivateAccessKey, obscure.MustReveal(encryptedPrivateAccessKey)),
}
authRequest := api.AppAuthorization{
Username: username,
Password: password,
Application: withDefault(opt.AppID, appID),
AccessKeyID: withDefault(opt.AccessKeyID, accessKeyID),
PrivateAccessKey: withDefault(opt.PrivateAccessKey, obscure.MustReveal(encryptedPrivateAccessKey)),
}
var resp *http.Response
opts := rest.Opts{
Method: "POST",
Path: "/app-authorization",
}
srv := rest.NewClient(fshttp.NewClient(ctx)).SetRoot(rootURL) // FIXME
var resp *http.Response
opts := rest.Opts{
Method: "POST",
Path: "/app-authorization",
}
srv := rest.NewClient(fshttp.NewClient(ctx)).SetRoot(rootURL) // FIXME
// FIXME
//err = f.pacer.Call(func() (bool, error) {
resp, err = srv.CallXML(context.Background(), &opts, &authRequest, nil)
// return shouldRetry(ctx, resp, err)
//})
if err != nil {
return errors.Wrap(err, "failed to get token")
// FIXME
//err = f.pacer.Call(func() (bool, error) {
resp, err = srv.CallXML(context.Background(), &opts, &authRequest, nil)
// return shouldRetry(ctx, resp, err)
//})
if err != nil {
return nil, errors.Wrap(err, "failed to get token")
}
opt.RefreshToken = resp.Header.Get("Location")
m.Set("refresh_token", opt.RefreshToken)
return nil, nil
}
opt.RefreshToken = resp.Header.Get("Location")
m.Set("refresh_token", opt.RefreshToken)
return nil
},
Options: []fs.Option{{
return nil, fmt.Errorf("unknown state %q", config.State)
}, Options: []fs.Option{{
Name: "app_id",
Help: "Sugarsync App ID.\n\nLeave blank to use rclone's.",
}, {

24
backend/tardigrade/fs.go
View File

@ -41,19 +41,19 @@ func init() {
Name: "tardigrade",
Description: "Tardigrade Decentralized Cloud Storage",
NewFs: NewFs,
Config: func(ctx context.Context, name string, configMapper configmap.Mapper) error {
provider, _ := configMapper.Get(fs.ConfigProvider)
Config: func(ctx context.Context, name string, m configmap.Mapper, configIn fs.ConfigIn) (*fs.ConfigOut, error) {
provider, _ := m.Get(fs.ConfigProvider)
config.FileDeleteKey(name, fs.ConfigProvider)
if provider == newProvider {
satelliteString, _ := configMapper.Get("satellite_address")
apiKey, _ := configMapper.Get("api_key")
passphrase, _ := configMapper.Get("passphrase")
satelliteString, _ := m.Get("satellite_address")
apiKey, _ := m.Get("api_key")
passphrase, _ := m.Get("passphrase")
// satelliteString contains always default and passphrase can be empty
if apiKey == "" {
return nil
return nil, nil
}
satellite, found := satMap[satelliteString]
@ -63,23 +63,23 @@ func init() {
access, err := uplink.RequestAccessWithPassphrase(context.TODO(), satellite, apiKey, passphrase)
if err != nil {
return errors.Wrap(err, "couldn't create access grant")
return nil, errors.Wrap(err, "couldn't create access grant")
}
serializedAccess, err := access.Serialize()
if err != nil {
return errors.Wrap(err, "couldn't serialize access grant")
return nil, errors.Wrap(err, "couldn't serialize access grant")
}
configMapper.Set("satellite_address", satellite)
configMapper.Set("access_grant", serializedAccess)
m.Set("satellite_address", satellite)
m.Set("access_grant", serializedAccess)
} else if provider == existingProvider {
config.FileDeleteKey(name, "satellite_address")
config.FileDeleteKey(name, "api_key")
config.FileDeleteKey(name, "passphrase")
} else {
return errors.Errorf("invalid provider type: %s", provider)
return nil, errors.Errorf("invalid provider type: %s", provider)
}
return nil
return nil, nil
},
Options: []fs.Option{
{

10
backend/yandex/yandex.go
View File

@ -60,12 +60,10 @@ func init() {
Name: "yandex",
Description: "Yandex Disk",
NewFs: NewFs,
Config: func(ctx context.Context, name string, m configmap.Mapper) error {
err := oauthutil.Config(ctx, "yandex", name, m, oauthConfig, nil)
if err != nil {
return errors.Wrap(err, "failed to configure token")
}
return nil
Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
return oauthutil.ConfigOut("", &oauthutil.Options{
OAuth2Config: oauthConfig,
})
},
Options: append(oauthutil.SharedOptions, []fs.Option{{
Name: config.ConfigEncoding,

141
backend/zoho/zoho.go
View File

@ -72,41 +72,89 @@ func init() {
Name: "zoho",
Description: "Zoho",
NewFs: NewFs,
Config: func(ctx context.Context, name string, m configmap.Mapper) error {
Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
// Need to setup region before configuring oauth
err := setupRegion(m)
if err != nil {
return err
return nil, err
}
opt := oauthutil.Options{
// No refresh token unless ApprovalForce is set
OAuth2Opts: []oauth2.AuthCodeOption{oauth2.ApprovalForce},
}
if err := oauthutil.Config(ctx, "zoho", name, m, oauthConfig, &opt); err != nil {
return errors.Wrap(err, "failed to configure token")
}
// We need to rewrite the token type to "Zoho-oauthtoken" because Zoho wants
// it's own custom type
token, err := oauthutil.GetToken(name, m)
if err != nil {
return errors.Wrap(err, "failed to read token")
}
if token.TokenType != "Zoho-oauthtoken" {
token.TokenType = "Zoho-oauthtoken"
err = oauthutil.PutToken(name, m, token, false)
getSrvs := func() (authSrv, apiSrv *rest.Client, err error) {
oAuthClient, _, err := oauthutil.NewClient(ctx, name, m, oauthConfig)
if err != nil {
return errors.Wrap(err, "failed to configure token")
return nil, nil, errors.Wrap(err, "failed to load oAuthClient")
}
authSrv = rest.NewClient(oAuthClient).SetRoot(accountsURL)
apiSrv = rest.NewClient(oAuthClient).SetRoot(rootURL)
return authSrv, apiSrv, nil
}
if fs.GetConfig(ctx).AutoConfirm {
return nil
}
switch config.State {
case "":
return oauthutil.ConfigOut("teams", &oauthutil.Options{
OAuth2Config: oauthConfig,
// No refresh token unless ApprovalForce is set
OAuth2Opts: []oauth2.AuthCodeOption{oauth2.ApprovalForce},
})
case "teams":
// We need to rewrite the token type to "Zoho-oauthtoken" because Zoho wants
// it's own custom type
token, err := oauthutil.GetToken(name, m)
if err != nil {
return nil, errors.Wrap(err, "failed to read token")
}
if token.TokenType != "Zoho-oauthtoken" {
token.TokenType = "Zoho-oauthtoken"
err = oauthutil.PutToken(name, m, token, false)
if err != nil {
return nil, errors.Wrap(err, "failed to configure token")
}
}
if err = setupRoot(ctx, name, m); err != nil {
return errors.Wrap(err, "failed to configure root directory")
authSrv, apiSrv, err := getSrvs()
if err != nil {
return nil, err
}
// Get the user Info
opts := rest.Opts{
Method: "GET",
Path: "/oauth/user/info",
}
var user api.User
_, err = authSrv.CallJSON(ctx, &opts, nil, &user)
if err != nil {
return nil, err
}
// Get the teams
teams, err := listTeams(ctx, user.ZUID, apiSrv)
if err != nil {
return nil, err
}
return fs.ConfigChoose("workspace", "Team Drive ID", len(teams), func(i int) (string, string) {
team := teams[i]
return team.ID, team.Attributes.Name
})
case "workspace":
_, apiSrv, err := getSrvs()
if err != nil {
return nil, err
}
teamID := config.Result
workspaces, err := listWorkspaces(ctx, teamID, apiSrv)
if err != nil {
return nil, err
}
return fs.ConfigChoose("workspace_end", "Workspace ID", len(workspaces), func(i int) (string, string) {
workspace := workspaces[i]
return workspace.ID, workspace.Attributes.Name
})
case "workspace_end":
worksspaceID := config.Result
m.Set(configRootID, worksspaceID)
return nil, nil
}
return nil
return nil, fmt.Errorf("unknown state %q", config.State)
},
Options: append(oauthutil.SharedOptions, []fs.Option{{
Name: "region",
@ -209,49 +257,6 @@ func listWorkspaces(ctx context.Context, teamID string, srv *rest.Client) ([]api
return workspaceList.TeamWorkspace, nil
}
func setupRoot(ctx context.Context, name string, m configmap.Mapper) error {
oAuthClient, _, err := oauthutil.NewClient(ctx, name, m, oauthConfig)
if err != nil {
return errors.Wrap(err, "failed to load oAuthClient")
}
authSrv := rest.NewClient(oAuthClient).SetRoot(accountsURL)
opts := rest.Opts{
Method: "GET",
Path: "/oauth/user/info",
}
var user api.User
_, err = authSrv.CallJSON(ctx, &opts, nil, &user)
if err != nil {
return err
}
apiSrv := rest.NewClient(oAuthClient).SetRoot(rootURL)
teams, err := listTeams(ctx, user.ZUID, apiSrv)
if err != nil {
return err
}
var teamIDs, teamNames []string
for _, team := range teams {
teamIDs = append(teamIDs, team.ID)
teamNames = append(teamNames, team.Attributes.Name)
}
teamID := config.Choose("Enter a Team Drive ID", teamIDs, teamNames, true)
workspaces, err := listWorkspaces(ctx, teamID, apiSrv)
if err != nil {
return err
}
var workspaceIDs, workspaceNames []string
for _, workspace := range workspaces {
workspaceIDs = append(workspaceIDs, workspace.ID)
workspaceNames = append(workspaceNames, workspace.Attributes.Name)
}
worksspaceID := config.Choose("Enter a Workspace ID", workspaceIDs, workspaceNames, true)
m.Set(configRootID, worksspaceID)
return nil
}
// --------------------------------------------------------------
// retryErrorCodes is a slice of error codes that we will retry

7
cmd/config/config.go
View File

@ -265,14 +265,11 @@ This normally means going through the interactive oauth flow again.
RunE: func(command *cobra.Command, args []string) error {
ctx := context.Background()
cmd.CheckArgs(1, 1, command, args)
fsInfo, configName, _, config, err := fs.ConfigFs(args[0])
fsInfo, configName, _, m, err := fs.ConfigFs(args[0])
if err != nil {
return err
}
if fsInfo.Config == nil {
return errors.Errorf("%s: doesn't support Reconnect", configName)
}
return fsInfo.Config(ctx, configName, config)
return config.PostConfig(ctx, configName, m, fsInfo)
},
}

277
fs/backend_config.go Normal file
View File

@ -0,0 +1,277 @@
// Structures and utilities for backend config
//
//
package fs
import (
"context"
"fmt"
"strings"
"github.com/pkg/errors"
"github.com/rclone/rclone/fs/config/configmap"
)
const (
// ConfigToken is the key used to store the token under
ConfigToken = "token"
)
// ConfigOAuth should be called to do the OAuth
//
// set in lib/oauthutil to avoid a circular import
var ConfigOAuth func(ctx context.Context, name string, m configmap.Mapper, ri *RegInfo, in ConfigIn) (*ConfigOut, error)
// ConfigIn is passed to the Config function for an Fs
//
// The interactive config system for backends is state based. This is so that different frontends to the config can be attached, eg over the API or web page.
//
// Each call to the config system supplies ConfigIn which tells the
// system what to do. Each will return a ConfigOut which gives a
// question to ask the user and a state to return to. There is one
// special question which allows the backends to do OAuth.
//
// The ConfigIn contains a State which the backend should act upon and
// a Result from the previous question to the user.
//
// If ConfigOut is nil or ConfigOut.State == "" then the process is
// deemed to have finished. If there is no Option in ConfigOut then
// the next state will be called immediately. This is wrapped in
// ConfigGoto and ConfigResult.
//
// Backends should keep no state in memory - if they need to persist
// things between calls it should be persisted in the config file.
// Things can also be persisted in the state using the StatePush and
// StatePop utilities here.
//
// The utilities here are convenience methods for different kinds of
// questions and responses.
type ConfigIn struct {
State string // State to run
Result string // Result from previous Option
}
// ConfigOut is returned from Config function for an Fs
//
// State is the state for the next call to Config
// OAuth is a special value set by oauthutil.ConfigOAuth
// Error is displayed to the user before asking a question
// Result is passed to the next call to Config if Option/OAuth isn't set
type ConfigOut struct {
State string // State to jump to after this
Option *Option // Option to query user about
OAuth interface{} `json:"-"` // Do OAuth if set
Error string // error to be displayed to the user
Result string // if Option/OAuth not set then this is passed to the next state
}
// ConfigInput asks the user for a string
//
// state should be the next state required
// help should be the help shown to the user
func ConfigInput(state string, help string) (*ConfigOut, error) {
return &ConfigOut{
State: state,
Option: &Option{
Help: help,
Default: "",
},
}, nil
}
// ConfigPassword asks the user for a password
//
// state should be the next state required
// help should be the help shown to the user
func ConfigPassword(state string, help string) (*ConfigOut, error) {
return &ConfigOut{
State: state,
Option: &Option{
Help: help,
Default: "",
IsPassword: true,
},
}, nil
}
// ConfigGoto goes to the next state with empty Result
//
// state should be the next state required
func ConfigGoto(state string) (*ConfigOut, error) {
return &ConfigOut{
State: state,
}, nil
}
// ConfigResult goes to the next state with result given
//
// state should be the next state required
// result should be the result for the next state
func ConfigResult(state, result string) (*ConfigOut, error) {
return &ConfigOut{
State: state,
Result: result,
}, nil
}
// ConfigError shows the error to the user and goes to the state passed in
//
// state should be the next state required
// Error should be the error shown to the user
func ConfigError(state string, Error string) (*ConfigOut, error) {
return &ConfigOut{
State: state,
Error: Error,
}, nil
}
// ConfigConfirm returns a ConfigOut structure which asks a Yes/No question
//
// state should be the next state required
// Default should be the default state
// help should be the help shown to the user
func ConfigConfirm(state string, Default bool, help string) (*ConfigOut, error) {
return &ConfigOut{
State: state,
Option: &Option{
Help: help,
Default: Default,
Examples: []OptionExample{{
Value: "true",
Help: "Yes",
}, {
Value: "false",
Help: "No",
}},
},
}, nil
}
// ConfigChooseFixed returns a ConfigOut structure which has a list of items to choose from.
//
// state should be the next state required
// help should be the help shown to the user
// items should be the items in the list
//
// It chooses the first item to be the default.
// If there are no items then it will return an error.
// If there is only one item it will short cut to the next state
func ConfigChooseFixed(state string, help string, items []OptionExample) (*ConfigOut, error) {
if len(items) == 0 {
return nil, errors.Errorf("no items found in: %s", help)
}
choose := &ConfigOut{
State: state,
Option: &Option{
Help: help,
Examples: items,
},
}
choose.Option.Default = choose.Option.Examples[0].Value
if len(items) == 1 {
// short circuit asking the question if only one entry
choose.Result = choose.Option.Examples[0].Value
choose.Option = nil
}
return choose, nil
}
// ConfigChoose returns a ConfigOut structure which has a list of items to choose from.
//
// state should be the next state required
// help should be the help shown to the user
// n should be the number of items in the list
// getItem should return the items (value, help)
//
// It chooses the first item to be the default.
// If there are no items then it will return an error.
// If there is only one item it will short cut to the next state
func ConfigChoose(state string, help string, n int, getItem func(i int) (itemValue string, itemHelp string)) (*ConfigOut, error) {
items := make(OptionExamples, n)
for i := range items {
items[i].Value, items[i].Help = getItem(i)
}
return ConfigChooseFixed(state, help, items)
}
// StatePush pushes a new values onto the front of the config string
func StatePush(state string, values ...string) string {
for i := range values {
values[i] = strings.Replace(values[i], ",", "，", -1) // replace comma with unicode wide version
}
if state != "" {
values = append(values[:len(values):len(values)], state)
}
return strings.Join(values, ",")
}
type configOAuthKeyType struct{}
// OAuth key for config
var configOAuthKey = configOAuthKeyType{}
// ConfigOAuthOnly marks the ctx so that the Config will stop after
// finding an OAuth
func ConfigOAuthOnly(ctx context.Context) context.Context {
return context.WithValue(ctx, configOAuthKey, struct{}{})
}
// Return true if ctx is marked as ConfigOAuthOnly
func isConfigOAuthOnly(ctx context.Context) bool {
return ctx.Value(configOAuthKey) != nil
}
// StatePop pops a state from the front of the config string
// It returns the new state and the value popped
func StatePop(state string) (newState string, value string) {
comma := strings.IndexRune(state, ',')
if comma < 0 {
return "", state
}
value, newState = state[:comma], state[comma+1:]
value = strings.Replace(value, "，", ",", -1) // replace unicode wide comma with comma
return newState, value
}
// BackendConfig calls the config for the backend in ri
//
// It wraps any OAuth transactions as necessary so only straight forward config questions are emitted
func BackendConfig(ctx context.Context, name string, m configmap.Mapper, ri *RegInfo, in ConfigIn) (*ConfigOut, error) {
ci := GetConfig(ctx)
if ri.Config == nil {
return nil, nil
}
// Do internal states here
if strings.HasPrefix(in.State, "*") {
switch {
case strings.HasPrefix(in.State, "*oauth"):
return ConfigOAuth(ctx, name, m, ri, in)
default:
return nil, errors.Errorf("unknown internal state %q", in.State)
}
}
out, err := ri.Config(ctx, name, m, in)
if err != nil {
return nil, err
}
switch {
case out == nil:
case out.OAuth != nil:
// If this is an OAuth state the deal with it here
returnState := out.State
// If rclone authorize, stop after doing oauth
if isConfigOAuthOnly(ctx) {
Debugf(nil, "OAuth only is set - overriding return state")
returnState = ""
}
// Run internal state, saving the input so we can recall the state
return ConfigGoto(StatePush("", "*oauth", returnState, in.State, in.Result))
case out.Option != nil && ci.AutoConfirm:
// If AutoConfirm is set, choose the default value
result := fmt.Sprint(out.Option.Default)
Debugf(nil, "Auto confirm is set, choosing default %q for state %q", result, out.State)
return ConfigResult(out.State, result)
}
return out, nil
}

37
fs/backend_config_test.go Normal file
View File

@ -0,0 +1,37 @@
package fs
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestStatePush(t *testing.T) {
assert.Equal(t, "", StatePush(""))
assert.Equal(t, "", StatePush("", ""))
assert.Equal(t, "a", StatePush("", "a"))
assert.Equal(t, "a,1，2，3", StatePush("", "a", "1,2,3"))
assert.Equal(t, "potato", StatePush("potato"))
assert.Equal(t, ",potato", StatePush("potato", ""))
assert.Equal(t, "a,potato", StatePush("potato", "a"))
assert.Equal(t, "a,1，2，3,potato", StatePush("potato", "a", "1,2,3"))
}
func TestStatePop(t *testing.T) {
state, value := StatePop("")
assert.Equal(t, "", value)
assert.Equal(t, "", state)
state, value = StatePop("a")
assert.Equal(t, "a", value)
assert.Equal(t, "", state)
state, value = StatePop("a,1，2，3")
assert.Equal(t, "a", value)
assert.Equal(t, "1，2，3", state)
state, value = StatePop("1，2，3,a")
assert.Equal(t, "1,2,3", value)
assert.Equal(t, "a", state)
}

6
fs/config/authorize.go
View File

@ -11,12 +11,14 @@ import (
// Authorize is for remote authorization of headless machines.
//
// It expects 1 or 3 arguments
// It expects 1, 2 or 3 arguments
//
// rclone authorize "fs name"
// rclone authorize "fs name" "base64 encoded JSON blob"
// rclone authorize "fs name" "client id" "client secret"
func Authorize(ctx context.Context, args []string, noAutoBrowser bool) error {
ctx = suppressConfirm(ctx)
ctx = fs.ConfigOAuthOnly(ctx)
switch len(args) {
case 1, 2, 3:
default:
@ -60,7 +62,7 @@ func Authorize(ctx context.Context, args []string, noAutoBrowser bool) error {
m.AddSetter(outM)
m.AddGetter(outM, configmap.PriorityNormal)
err = ri.Config(ctx, name, m)
err = PostConfig(ctx, name, m, ri)
if err != nil {
return err
}

95
fs/config/ui.go
View File

@ -90,34 +90,6 @@ func Confirm(Default bool) bool {
return CommandDefault([]string{"yYes", "nNo"}, defaultIndex) == 'y'
}
// ConfirmWithConfig asks the user for Yes or No and returns true or
// false.
//
// If AutoConfirm is set, it will look up the value in m and return
// that, but if it isn't set then it will return the Default value
// passed in
func ConfirmWithConfig(ctx context.Context, m configmap.Getter, configName string, Default bool) bool {
ci := fs.GetConfig(ctx)
if ci.AutoConfirm {
configString, ok := m.Get(configName)
if ok {
configValue, err := strconv.ParseBool(configString)
if err != nil {
fs.Errorf(nil, "Failed to parse config parameter %s=%q as boolean - using default %v: %v", configName, configString, Default, err)
} else {
Default = configValue
}
}
answer := "No"
if Default {
answer = "Yes"
}
fmt.Printf("Auto confirm is set: answering %s, override by setting config parameter %s=%v\n", answer, configName, !Default)
return Default
}
return Confirm(Default)
}
// Choose one of the defaults or type a new string if newOk is set
func Choose(what string, defaults, help []string, newOk bool) string {
valueDescription := "an existing"
@ -269,15 +241,72 @@ func OkRemote(name string) bool {
return false
}
// PostConfig configures the backend after the main config has been done
//
// The is the user interface loop that drives the post configuration backend config.
func PostConfig(ctx context.Context, name string, m configmap.Mapper, ri *fs.RegInfo) error {
// FIXME if doing authorize, stop when we've got to the OAuth
if ri.Config == nil {
return errors.New("backend doesn't support reconnect or authorize")
}
in := fs.ConfigIn{
State: "",
}
for {
fs.Debugf(name, "config: state=%q, result=%q", in.State, in.Result)
out, err := fs.BackendConfig(ctx, name, m, ri, in)
if err != nil {
return err
}
if out == nil {
break
}
if out.Error != "" {
fmt.Println(out.Error)
}
in.State = out.State
in.Result = out.Result
if out.Option != nil {
if out.Option.Default == nil {
out.Option.Default = ""
}
if Default, isBool := out.Option.Default.(bool); isBool &&
len(out.Option.Examples) == 2 &&
out.Option.Examples[0].Help == "Yes" &&
out.Option.Examples[0].Value == "true" &&
out.Option.Examples[1].Help == "No" &&
out.Option.Examples[1].Value == "false" &&
out.Option.Exclusive {
// Use Confirm for Yes/No questions as it has a nicer interface=
fmt.Println(out.Option.Help)
in.Result = fmt.Sprint(Confirm(Default))
} else {
value := ChooseOption(out.Option, "")
if value != "" {
err := out.Option.Set(value)
if err != nil {
return errors.Wrap(err, "failed to set option")
}
}
in.Result = out.Option.String()
}
}
if out.State == "" {
break
}
}
return nil
}
// RemoteConfig runs the config helper for the remote if needed
func RemoteConfig(ctx context.Context, name string) error {
fmt.Printf("Remote config\n")
f := mustFindByName(name)
if f.Config != nil {
m := fs.ConfigMap(f, name, nil)
return f.Config(ctx, name, m)
ri := mustFindByName(name)
m := fs.ConfigMap(ri, name, nil)
if ri.Config == nil {
return nil
}
return nil
return PostConfig(ctx, name, m, ri)
}
// matchProvider returns true if provider matches the providerConfig string.

4
fs/fs.go
View File

@ -88,8 +88,8 @@ type RegInfo struct {
// object, then it should return an Fs which which points to
// the parent of that object and ErrorIsFile.
NewFs func(ctx context.Context, name string, root string, config configmap.Mapper) (Fs, error) `json:"-"`
// Function to call to help with config
Config func(ctx context.Context, name string, config configmap.Mapper) error `json:"-"`
// Function to call to help with config - see docs for ConfigIn for more info
Config func(ctx context.Context, name string, m configmap.Mapper, configIn ConfigIn) (*ConfigOut, error) `json:"-"`
// Options for the Fs configuration
Options Options
// The command help, if any

356
lib/oauthutil/oauthutil.go
View File

@ -8,6 +8,7 @@ import (
"net"
"net/http"
"net/url"
"strings"
"sync"
"time"
@ -393,68 +394,94 @@ type CheckAuthFn func(*oauth2.Config, *AuthResult) error
// Options for the oauth config
type Options struct {
OAuth2Config *oauth2.Config // Basic config for oauth2
NoOffline bool // If set then "access_type=offline" parameter is not passed
CheckAuth CheckAuthFn // When the AuthResult is known the checkAuth function is called if set
OAuth2Opts []oauth2.AuthCodeOption // extra oauth2 options
StateBlankOK bool // If set, state returned as "" is deemed to be OK
}
// Config does the initial creation of the token
// ConfigOut returns a config item suitable for the backend config
//
// If opt is nil it will use the default Options
// state is the place to return the config to
// oAuth is the config to run the oauth with
func ConfigOut(state string, oAuth *Options) (*fs.ConfigOut, error) {
return &fs.ConfigOut{
State: state,
OAuth: oAuth,
}, nil
}
// ConfigOAuth does the oauth config specified in the config block
//
// It may run an internal webserver to receive the results
func Config(ctx context.Context, id, name string, m configmap.Mapper, oauthConfig *oauth2.Config, opt *Options) error {
if opt == nil {
opt = &Options{}
}
oauthConfig, changed := overrideCredentials(name, m, oauthConfig)
authorizeOnlyValue, ok := m.Get(config.ConfigAuthorize)
authorizeOnly := ok && authorizeOnlyValue != "" // set if being run by "rclone authorize"
authorizeNoAutoBrowserValue, ok := m.Get(config.ConfigAuthNoBrowser)
authorizeNoAutoBrowser := ok && authorizeNoAutoBrowserValue != ""
// This is called with a state which has pushed on it
//
// state prefixed with "*oauth"
// state for oauth to return to
// state that returned the OAuth when we wish to recall it
// value that returned the OAuth
func ConfigOAuth(ctx context.Context, name string, m configmap.Mapper, ri *fs.RegInfo, in fs.ConfigIn) (*fs.ConfigOut, error) {
stateParams, state := fs.StatePop(in.State)
// See if already have a token
tokenString, ok := m.Get("token")
if ok && tokenString != "" {
fmt.Printf("Already have a token - refresh?\n")
if !config.ConfirmWithConfig(ctx, m, "config_refresh_token", true) {
return nil
}
// Make the next state
newState := func(state string) string {
return fs.StatePush(stateParams, state)
}
// Ask the user whether they are using a local machine
isLocal := func() bool {
fmt.Printf("Use auto config?\n")
fmt.Printf(" * Say Y if not sure\n")
fmt.Printf(" * Say N if you are working on a remote or headless machine\n")
return config.ConfirmWithConfig(ctx, m, "config_is_local", true)
// Recall the Oauth state again by calling the Config with the same input again
getOAuth := func() (opt *Options, err error) {
tmpState, _ := fs.StatePop(stateParams)
tmpState, State := fs.StatePop(tmpState)
_, Result := fs.StatePop(tmpState)
out, err := ri.Config(ctx, name, m, fs.ConfigIn{State: State, Result: Result})
if err != nil {
return nil, err
}
if out.OAuth == nil {
return nil, errors.New("failed to recall OAuth state")
}
opt, ok := out.OAuth.(*Options)
if !ok {
return nil, errors.Errorf("internal error: oauth failed: wrong type in config: %T", out.OAuth)
}
if opt.OAuth2Config == nil {
return nil, errors.New("internal error: oauth failed: OAuth2Config not set")
}
return opt, nil
}
// Detect whether we should use internal web server
useWebServer := false
switch oauthConfig.RedirectURL {
case TitleBarRedirectURL:
useWebServer = authorizeOnly
if !authorizeOnly {
useWebServer = isLocal()
switch state {
case "*oauth":
// See if already have a token
tokenString, ok := m.Get("token")
if ok && tokenString != "" {
return fs.ConfigConfirm(newState("*oauth-confirm"), true, "Already have a token - refresh?")
}
if useWebServer {
// copy the config and set to use the internal webserver
configCopy := *oauthConfig
oauthConfig = &configCopy
oauthConfig.RedirectURL = RedirectURL
return fs.ConfigGoto(newState("*oauth-confirm"))
case "*oauth-confirm":
if in.Result == "false" {
return fs.ConfigGoto(newState("*oauth-done"))
}
default:
if changed {
fmt.Printf("Make sure your Redirect URL is set to %q in your custom config.\n", oauthConfig.RedirectURL)
return fs.ConfigConfirm(newState("*oauth-islocal"), true, "Use auto config?\n * Say Y if not sure\n * Say N if you are working on a remote or headless machine\n")
case "*oauth-islocal":
if in.Result == "true" {
return fs.ConfigGoto(newState("*oauth-do"))
}
useWebServer = true
if authorizeOnly {
break
return fs.ConfigGoto(newState("*oauth-remote"))
case "*oauth-remote":
opt, err := getOAuth()
if err != nil {
return nil, err
}
if !isLocal() {
fmt.Printf(`For this to work, you will need rclone available on a machine that has
if noWebserverNeeded(opt.OAuth2Config) {
authURL, _, err := getAuthURL(name, m, opt.OAuth2Config, opt)
if err != nil {
return nil, err
}
return fs.ConfigInput(newState("*oauth-do"), fmt.Sprintf("Verification code\n\nGo to this URL, authenticate then paste the code here.\n\n%s\n", authURL))
}
var out strings.Builder
fmt.Fprintf(&out, `For this to work, you will need rclone available on a machine that has
a web browser available.
For more help and alternate methods see: https://rclone.org/remote_setup/
@ -463,66 +490,97 @@ Execute the following on the machine with the web browser (same rclone
version recommended):
`)
// Find the configuration
ri, err := fs.Find(id)
if err != nil {
return errors.Wrap(err, "oauthutil authorize")
}
// Find the overridden options
inM := ri.Options.NonDefault(m)
delete(inM, config.ConfigToken) // delete token as we are refreshing it
for k, v := range inM {
fs.Debugf(nil, "sending %s = %q", k, v)
}
// Encode them into a string
mCopyString, err := inM.Encode()
if err != nil {
return errors.Wrap(err, "oauthutil authorize encode")
}
// Write what the user has to do
useNewFormat := len(mCopyString) > 0
if useNewFormat {
fmt.Printf("\trclone authorize %q %q\n", id, mCopyString)
} else {
fmt.Printf("\trclone authorize %q\n", id)
}
fmt.Println("\nThen paste the result below:")
// Read the updates to the config
var outM configmap.Simple
var token oauth2.Token
for {
outM = configmap.Simple{}
token = oauth2.Token{}
code := config.ReadNonEmptyLine("result> ")
if useNewFormat {
err = outM.Decode(code)
} else {
err = json.Unmarshal([]byte(code), &token)
}
if err == nil {
break
}
fmt.Printf("Couldn't decode response - try again (make sure you are using a matching version of rclone on both sides: %v\n", err)
}
// Save the config updates
if useNewFormat {
for k, v := range outM {
m.Set(k, v)
fs.Debugf(nil, "received %s = %q", k, v)
}
return nil
}
return PutToken(name, m, &token, true)
// Find the overridden options
inM := ri.Options.NonDefault(m)
delete(inM, fs.ConfigToken) // delete token as we are refreshing it
for k, v := range inM {
fs.Debugf(nil, "sending %s = %q", k, v)
}
// Encode them into a string
mCopyString, err := inM.Encode()
if err != nil {
return nil, errors.Wrap(err, "oauthutil authorize encode")
}
// Write what the user has to do
if len(mCopyString) > 0 {
fmt.Fprintf(&out, "\trclone authorize %q %q\n", ri.Name, mCopyString)
} else {
fmt.Fprintf(&out, "\trclone authorize %q\n", ri.Name)
}
fmt.Fprintln(&out, "\nThen paste the result.")
return fs.ConfigInput(newState("*oauth-authorize"), out.String())
case "*oauth-authorize":
// Read the updates to the config
outM := configmap.Simple{}
token := oauth2.Token{}
code := in.Result
newFormat := true
err := outM.Decode(code)
if err != nil {
newFormat = false
err = json.Unmarshal([]byte(code), &token)
}
if err != nil {
return fs.ConfigError(newState("*oauth-authorize"), fmt.Sprintf("Couldn't decode response - try again (make sure you are using a matching version of rclone on both sides: %v\n", err))
}
// Save the config updates
if newFormat {
for k, v := range outM {
m.Set(k, v)
fs.Debugf(nil, "received %s = %q", k, v)
}
} else {
m.Set(fs.ConfigToken, code)
}
return fs.ConfigGoto(newState("*oauth-done"))
case "*oauth-do":
code := in.Result
opt, err := getOAuth()
if err != nil {
return nil, err
}
oauthConfig, changed := overrideCredentials(name, m, opt.OAuth2Config)
if changed {
fs.Logf(nil, "Make sure your Redirect URL is set to %q in your custom config.\n", oauthConfig.RedirectURL)
}
if code == "" {
oauthConfig = fixRedirect(oauthConfig)
code, err = configSetup(ctx, ri.Name, name, m, oauthConfig, opt)
if err != nil {
return nil, errors.Wrap(err, "config failed to refresh token")
}
}
err = configExchange(ctx, name, m, oauthConfig, code)
if err != nil {
return nil, err
}
return fs.ConfigGoto(newState("*oauth-done"))
case "*oauth-done":
// Return to the state indicated in the State stack
_, returnState := fs.StatePop(stateParams)
return fs.ConfigGoto(returnState)
}
return nil, errors.Errorf("unknown internal oauth state %q", state)
}
func init() {
// Set the function to avoid circular import
fs.ConfigOAuth = ConfigOAuth
}
// Return true if can run without a webserver and just entering a code
func noWebserverNeeded(oauthConfig *oauth2.Config) bool {
return oauthConfig.RedirectURL == TitleBarRedirectURL
}
// get the URL we need to send the user to
func getAuthURL(name string, m configmap.Mapper, oauthConfig *oauth2.Config, opt *Options) (authURL string, state string, err error) {
oauthConfig, _ = overrideCredentials(name, m, oauthConfig)
// Make random state
state, err := random.Password(128)
state, err = random.Password(128)
if err != nil {
return err
return "", "", err
}
// Generate oauth URL
@ -530,58 +588,82 @@ version recommended):
if !opt.NoOffline {
opts = append(opts, oauth2.AccessTypeOffline)
}
authURL := oauthConfig.AuthCodeURL(state, opts...)
authURL = oauthConfig.AuthCodeURL(state, opts...)
return authURL, state, nil
}
// Prepare webserver if needed
var server *authServer
if useWebServer {
server = newAuthServer(opt, bindAddress, state, authURL)
err := server.Init()
if err != nil {
return errors.Wrap(err, "failed to start auth webserver")
}
go server.Serve()
defer server.Stop()
authURL = "http://" + bindAddress + "/auth?state=" + state
// If TitleBarRedirect is set but we are doing a real oauth, then
// override our redirect URL
func fixRedirect(oauthConfig *oauth2.Config) *oauth2.Config {
switch oauthConfig.RedirectURL {
case TitleBarRedirectURL:
// copy the config and set to use the internal webserver
configCopy := *oauthConfig
oauthConfig = &configCopy
oauthConfig.RedirectURL = RedirectURL
}
return oauthConfig
}
// configSetup does the initial creation of the token
//
// If opt is nil it will use the default Options
//
// It will run an internal webserver to receive the results
func configSetup(ctx context.Context, id, name string, m configmap.Mapper, oauthConfig *oauth2.Config, opt *Options) (string, error) {
if opt == nil {
opt = &Options{}
}
authorizeNoAutoBrowserValue, ok := m.Get(config.ConfigAuthNoBrowser)
authorizeNoAutoBrowser := ok && authorizeNoAutoBrowserValue != ""
authURL, state, err := getAuthURL(name, m, oauthConfig, opt)
if err != nil {
return "", err
}
if !authorizeNoAutoBrowser && oauthConfig.RedirectURL != TitleBarRedirectURL {
// Prepare webserver
server := newAuthServer(opt, bindAddress, state, authURL)
err = server.Init()
if err != nil {
return "", errors.Wrap(err, "failed to start auth webserver")
}
go server.Serve()
defer server.Stop()
authURL = "http://" + bindAddress + "/auth?state=" + state
if !authorizeNoAutoBrowser {
// Open the URL for the user to visit
_ = open.Start(authURL)
fmt.Printf("If your browser doesn't open automatically go to the following link: %s\n", authURL)
fs.Logf(nil, "If your browser doesn't open automatically go to the following link: %s\n", authURL)
} else {
fmt.Printf("Please go to the following link: %s\n", authURL)
fs.Logf(nil, "Please go to the following link: %s\n", authURL)
}
fmt.Printf("Log in and authorize rclone for access\n")
fs.Logf(nil, "Log in and authorize rclone for access\n")
// Read the code via the webserver or manually
var auth *AuthResult
if useWebServer {
fmt.Printf("Waiting for code...\n")
auth = <-server.result
if !auth.OK || auth.Code == "" {
return auth
}
fmt.Printf("Got code\n")
if opt.CheckAuth != nil {
err = opt.CheckAuth(oauthConfig, auth)
if err != nil {
return err
}
}
} else {
auth = &AuthResult{
Code: config.ReadNonEmptyLine("Enter verification code> "),
// Read the code via the webserver
fs.Logf(nil, "Waiting for code...\n")
auth := <-server.result
if !auth.OK || auth.Code == "" {
return "", auth
}
fs.Logf(nil, "Got code\n")
if opt.CheckAuth != nil {
err = opt.CheckAuth(oauthConfig, auth)
if err != nil {
return "", err
}
}
return auth.Code, nil
}
// Exchange the code for a token
// Exchange the code for a token
func configExchange(ctx context.Context, name string, m configmap.Mapper, oauthConfig *oauth2.Config, code string) error {
ctx = Context(ctx, fshttp.NewClient(ctx))
token, err := oauthConfig.Exchange(ctx, auth.Code)
token, err := oauthConfig.Exchange(ctx, code)
if err != nil {
return errors.Wrap(err, "failed to get token")
}
return PutToken(name, m, token, true)
}