community
/
rclone
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

serve http: support --auth-proxy

This commit is contained in:
Matthias Baur 2023-02-22 15:55:24 +01:00 committed by GitHub
parent ce8b1cd861
commit a0b5d77427
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 159 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

97
cmd/serve/http/http.go
View File

@ -3,6 +3,7 @@ package http
import (
"context"
"errors"
"fmt"
"io"
"log"
@ -15,6 +16,8 @@ import (
"github.com/go-chi/chi/v5/middleware"
"github.com/rclone/rclone/cmd"
"github.com/rclone/rclone/cmd/serve/proxy"
"github.com/rclone/rclone/cmd/serve/proxy/proxyflags"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/accounting"
libhttp "github.com/rclone/rclone/lib/http"
@ -47,6 +50,7 @@ func init() {
libhttp.AddHTTPFlagsPrefix(flagSet, "", &Opt.HTTP)
libhttp.AddTemplateFlagsPrefix(flagSet, "", &Opt.Template)
vfsflags.AddFlags(flagSet)
proxyflags.AddFlags(flagSet)
}
// Command definition for cobra
@ -64,18 +68,21 @@ The server will log errors. Use ` + "`-v`" + ` to see access logs.
` + "`--bwlimit`" + ` will be respected for file transfers. Use ` + "`--stats`" + ` to
control the stats printing.
` + libhttp.Help + libhttp.TemplateHelp + libhttp.AuthHelp + vfs.Help,
` + libhttp.Help + libhttp.TemplateHelp + libhttp.AuthHelp + vfs.Help + proxy.Help,
Annotations: map[string]string{
"versionIntroduced": "v1.39",
},
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(1, 1, command, args)
f := cmd.NewFsSrc(args)
var f fs.Fs
if proxyflags.Opt.AuthProxy == "" {
cmd.CheckArgs(1, 1, command, args)
f = cmd.NewFsSrc(args)
} else {
cmd.CheckArgs(0, 0, command, args)
}
cmd.Run(false, true, command, func() error {
ctx := context.Background()
s, err := run(ctx, f, Opt)
s, err := run(context.Background(), f, Opt)
if err != nil {
log.Fatal(err)
}
@ -86,25 +93,60 @@ control the stats printing.
},
}
// server contains everything to run the server
type serveCmd struct {
// HTTP contains everything to run the server
type HTTP struct {
f fs.Fs
vfs *vfs.VFS
_vfs *vfs.VFS // don't use directly, use getVFS
server *libhttp.Server
opt Options
proxy *proxy.Proxy
ctx context.Context // for global config
}
func run(ctx context.Context, f fs.Fs, opt Options) (*serveCmd, error) {
var err error
// Gets the VFS in use for this request
func (s *HTTP) getVFS(ctx context.Context) (VFS *vfs.VFS, err error) {
if s._vfs != nil {
return s._vfs, nil
}
value := libhttp.CtxGetAuth(ctx)
if value == nil {
return nil, errors.New("no VFS found in context")
}
VFS, ok := value.(*vfs.VFS)
if !ok {
return nil, fmt.Errorf("context value is not VFS: %#v", value)
}
return VFS, nil
}
s := &serveCmd{
// auth does proxy authorization
func (s *HTTP) auth(user, pass string) (value interface{}, err error) {
VFS, _, err := s.proxy.Call(user, pass, false)
if err != nil {
return nil, err
}
return VFS, err
}
func run(ctx context.Context, f fs.Fs, opt Options) (s *HTTP, err error) {
s = &HTTP{
f: f,
vfs: vfs.New(f, &vfsflags.Opt),
ctx: ctx,
opt: opt,
}
if proxyflags.Opt.AuthProxy != "" {
s.proxy = proxy.New(ctx, &proxyflags.Opt)
// override auth
s.opt.Auth.CustomAuthFn = s.auth
} else {
s._vfs = vfs.New(f, &vfsflags.Opt)
}
s.server, err = libhttp.NewServer(ctx,
libhttp.WithConfig(opt.HTTP),
libhttp.WithAuth(opt.Auth),
libhttp.WithTemplate(opt.Template),
libhttp.WithConfig(s.opt.HTTP),
libhttp.WithAuth(s.opt.Auth),
libhttp.WithTemplate(s.opt.Template),
)
if err != nil {
return nil, fmt.Errorf("failed to init server: %w", err)
@ -124,7 +166,7 @@ func run(ctx context.Context, f fs.Fs, opt Options) (*serveCmd, error) {
}
// handler reads incoming requests and dispatches them
func (s *serveCmd) handler(w http.ResponseWriter, r *http.Request) {
func (s *HTTP) handler(w http.ResponseWriter, r *http.Request) {
isDir := strings.HasSuffix(r.URL.Path, "/")
remote := strings.Trim(r.URL.Path, "/")
if isDir {
@ -135,9 +177,15 @@ func (s *serveCmd) handler(w http.ResponseWriter, r *http.Request) {
}
// serveDir serves a directory index at dirRemote
func (s *serveCmd) serveDir(w http.ResponseWriter, r *http.Request, dirRemote string) {
func (s *HTTP) serveDir(w http.ResponseWriter, r *http.Request, dirRemote string) {
VFS, err := s.getVFS(r.Context())
if err != nil {
http.Error(w, "Root directory not found", http.StatusNotFound)
fs.Errorf(nil, "Failed to serve directory: %v", err)
return
}
// List the directory
node, err := s.vfs.Stat(dirRemote)
node, err := VFS.Stat(dirRemote)
if err == vfs.ENOENT {
http.Error(w, "Directory not found", http.StatusNotFound)
return
@ -177,8 +225,15 @@ func (s *serveCmd) serveDir(w http.ResponseWriter, r *http.Request, dirRemote st
}
// serveFile serves a file object at remote
func (s *serveCmd) serveFile(w http.ResponseWriter, r *http.Request, remote string) {
node, err := s.vfs.Stat(remote)
func (s *HTTP) serveFile(w http.ResponseWriter, r *http.Request, remote string) {
VFS, err := s.getVFS(r.Context())
if err != nil {
http.Error(w, "File not found", http.StatusNotFound)
fs.Errorf(nil, "Failed to serve file: %v", err)
return
}
node, err := VFS.Stat(remote)
if err == vfs.ENOENT {
fs.Infof(remote, "%s: File not found", r.RemoteAddr)
http.Error(w, "File not found", http.StatusNotFound)

2
cmd/serve/http/http_test.go
View File

@ -21,7 +21,7 @@ import (
var (
updateGolden = flag.Bool("updategolden", false, "update golden files for regression test")
sc *serveCmd
sc *HTTP
testURL string
)

82
docs/content/commands/rclone_serve_http.md
View File

@ -437,6 +437,87 @@ _WARNING._ Contrary to `rclone size`, this flag ignores filters so that the
result is accurate. However, this is very inefficient and may cost lots of API
calls resulting in extra charges. Use it as a last resort and only with caching.
## Auth Proxy
If you supply the parameter `--auth-proxy /path/to/program` then
rclone will use that program to generate backends on the fly which
then are used to authenticate incoming requests. This uses a simple
JSON based protocol with input on STDIN and output on STDOUT.
**PLEASE NOTE:** `--auth-proxy` and `--authorized-keys` cannot be used
together, if `--auth-proxy` is set the authorized keys option will be
ignored.
There is an example program
[bin/test_proxy.py](https://github.com/rclone/rclone/blob/master/test_proxy.py)
in the rclone source code.
The program's job is to take a `user` and `pass` on the input and turn
those into the config for a backend on STDOUT in JSON format. This
config will have any default parameters for the backend added, but it
won't use configuration from environment variables or command line
options - it is the job of the proxy program to make a complete
config.
This config generated must have this extra parameter
- `_root` - root to use for the backend
And it may have this parameter
- `_obscure` - comma separated strings for parameters to obscure
If password authentication was used by the client, input to the proxy
process (on STDIN) would look similar to this:
```
{
"user": "me",
"pass": "mypassword"
}
```
If public-key authentication was used by the client, input to the
proxy process (on STDIN) would look similar to this:
```
{
"user": "me",
"public_key": "AAAAB3NzaC1yc2EAAAADAQABAAABAQDuwESFdAe14hVS6omeyX7edc...JQdf"
}
```
And as an example return this on STDOUT
```
{
"type": "sftp",
"_root": "",
"_obscure": "pass",
"user": "me",
"pass": "mypassword",
"host": "sftp.example.com"
}
```
This would mean that an SFTP backend would be created on the fly for
the `user` and `pass`/`public_key` returned in the output to the host given. Note
that since `_obscure` is set to `pass`, rclone will obscure the `pass`
parameter before creating the backend (which is required for sftp
backends).
The program can manipulate the supplied `user` in any way, for example
to make proxy to many different sftp backends, you could make the
`user` be `user@example.com` and then set the `host` to `example.com`
in the output and the user to `user`. For security you'd probably want
to restrict the `host` to a limited list.
Note that an internal cache is keyed on `user` so only use that for
configuration, don't use `pass` or `public_key`. This also means that if a user's
password or public-key is changed the cache will need to expire (which takes 5 mins)
before it takes effect.
This can be used to build general purpose proxies to any kind of
backend that rclone supports.
```
rclone serve http remote:path [flags]
@ -446,6 +527,7 @@ rclone serve http remote:path [flags]
```
--addr stringArray IPaddress:Port or :Port to bind server to (default [127.0.0.1:8080])
--auth-proxy string A program to use to create the backend from the auth
--baseurl string Prefix for URLs - leave blank for root
--cert string TLS PEM key (concatenation of certificate and CA certificate)
--client-ca string Client certificate authority to verify clients with