diff --git a/backend/all/all.go b/backend/all/all.go index fc9dd1791..cad76f6ad 100644 --- a/backend/all/all.go +++ b/backend/all/all.go @@ -37,6 +37,7 @@ import ( _ "github.com/rclone/rclone/backend/seafile" _ "github.com/rclone/rclone/backend/sftp" _ "github.com/rclone/rclone/backend/sharefile" + _ "github.com/rclone/rclone/backend/sia" _ "github.com/rclone/rclone/backend/sugarsync" _ "github.com/rclone/rclone/backend/swift" _ "github.com/rclone/rclone/backend/tardigrade" diff --git a/backend/sia/api/types.go b/backend/sia/api/types.go new file mode 100644 index 000000000..ebec8961e --- /dev/null +++ b/backend/sia/api/types.go @@ -0,0 +1,98 @@ +package api + +import ( + "strings" + "time" +) + +// DirectoriesResponse is the response for https://sia.tech/docs/#renter-dir-siapath-get +type DirectoriesResponse struct { + Directories []DirectoryInfo `json:"directories"` + Files []FileInfo `json:"files"` +} + +// FilesResponse is the response for https://sia.tech/docs/#renter-files-get +type FilesResponse struct { + Files []FileInfo `json:"files"` +} + +// FileResponse is the response for https://sia.tech/docs/#renter-file-siapath-get +type FileResponse struct { + File FileInfo `json:"file"` +} + +// FileInfo is used in https://sia.tech/docs/#renter-files-get +type FileInfo struct { + AccessTime time.Time `json:"accesstime"` + Available bool `json:"available"` + ChangeTime time.Time `json:"changetime"` + CipherType string `json:"ciphertype"` + CreateTime time.Time `json:"createtime"` + Expiration uint64 `json:"expiration"` + Filesize uint64 `json:"filesize"` + Health float64 `json:"health"` + LocalPath string `json:"localpath"` + MaxHealth float64 `json:"maxhealth"` + MaxHealthPercent float64 `json:"maxhealthpercent"` + ModTime time.Time `json:"modtime"` + NumStuckChunks uint64 `json:"numstuckchunks"` + OnDisk bool `json:"ondisk"` + Recoverable bool `json:"recoverable"` + Redundancy float64 `json:"redundancy"` + Renewing bool `json:"renewing"` + SiaPath string `json:"siapath"` + Stuck bool `json:"stuck"` + StuckHealth float64 `json:"stuckhealth"` + UploadedBytes uint64 `json:"uploadedbytes"` + UploadProgress float64 `json:"uploadprogress"` +} + +// DirectoryInfo is used in https://sia.tech/docs/#renter-dir-siapath-get +type DirectoryInfo struct { + AggregateHealth float64 `json:"aggregatehealth"` + AggregateLastHealthCheckTime time.Time `json:"aggregatelasthealthchecktime"` + AggregateMaxHealth float64 `json:"aggregatemaxhealth"` + AggregateMaxHealthPercentage float64 `json:"aggregatemaxhealthpercentage"` + AggregateMinRedundancy float64 `json:"aggregateminredundancy"` + AggregateMostRecentModTime time.Time `json:"aggregatemostrecentmodtime"` + AggregateNumFiles uint64 `json:"aggregatenumfiles"` + AggregateNumStuckChunks uint64 `json:"aggregatenumstuckchunks"` + AggregateNumSubDirs uint64 `json:"aggregatenumsubdirs"` + AggregateSize uint64 `json:"aggregatesize"` + AggregateStuckHealth float64 `json:"aggregatestuckhealth"` + + Health float64 `json:"health"` + LastHealthCheckTime time.Time `json:"lasthealthchecktime"` + MaxHealthPercentage float64 `json:"maxhealthpercentage"` + MaxHealth float64 `json:"maxhealth"` + MinRedundancy float64 `json:"minredundancy"` + MostRecentModTime time.Time `json:"mostrecentmodtime"` + NumFiles uint64 `json:"numfiles"` + NumStuckChunks uint64 `json:"numstuckchunks"` + NumSubDirs uint64 `json:"numsubdirs"` + SiaPath string `json:"siapath"` + Size uint64 `json:"size"` + StuckHealth float64 `json:"stuckhealth"` +} + +// Error contains an error message per https://sia.tech/docs/#error +type Error struct { + Message string `json:"message"` + Status string + StatusCode int +} + +// Error returns a string for the error and satisfies the error interface +func (e *Error) Error() string { + var out []string + if e.Message != "" { + out = append(out, e.Message) + } + if e.Status != "" { + out = append(out, e.Status) + } + if len(out) == 0 { + return "Siad Error" + } + return strings.Join(out, ": ") +} diff --git a/backend/sia/sia.go b/backend/sia/sia.go new file mode 100644 index 000000000..11c0c2323 --- /dev/null +++ b/backend/sia/sia.go @@ -0,0 +1,478 @@ +package sia + +import ( + "context" + "encoding/json" + "fmt" + "github.com/rclone/rclone/fs/config" + "github.com/rclone/rclone/lib/encoder" + "io" + "net/http" + "net/url" + "path" + "strings" + "time" + + "github.com/pkg/errors" + "github.com/rclone/rclone/backend/sia/api" + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/config/configmap" + "github.com/rclone/rclone/fs/config/configstruct" + "github.com/rclone/rclone/fs/config/obscure" + "github.com/rclone/rclone/fs/fserrors" + "github.com/rclone/rclone/fs/fshttp" + "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/lib/pacer" + "github.com/rclone/rclone/lib/rest" +) + +const ( + minSleep = 10 * time.Millisecond + maxSleep = 2 * time.Second + decayConstant = 2 // bigger for slower decay, exponential +) + +// Register with Fs +func init() { + fs.Register(&fs.RegInfo{ + Name: "sia", + Description: "Sia Decentralized Cloud", + NewFs: NewFs, + Options: []fs.Option{{ + Name: "api_url", + Help: "Sia HTTP API URL\nLike http://127.0.0.1:9980", + Required: true, + }, { + Name: "api_password", + Help: "Sia API Password\nsiad API Password", + Required: false, + IsPassword: true, + }, { + Name: "sia_user_agent", + Help: "Siad User Agent\nSia requires a 'Sia-Agent' user agent by default for security", + Required: false, + Default: "Sia-Agent", + Advanced: true, + }, { + Name: config.ConfigEncoding, + Help: config.ConfigEncodingHelp, + Advanced: true, + Default: encoder.EncodeInvalidUtf8 | + encoder.EncodeSlash, + }, + }}) +} + +// Options defines the configuration for this backend +type Options struct { + APIURL string `config:"api_url"` + APIPassword string `config:"api_password"` + UserAgent string `config:"sia_user_agent"` + Enc encoder.MultiEncoder `config:"encoding"` +} + +// Fs represents a remote siad +type Fs struct { + name string // name of this remote + root string // the path we are working on if any + opt Options // parsed config options + features *fs.Features // optional features + srv *rest.Client // the connection to siad + pacer *fs.Pacer // pacer for API calls +} + +// Object describes a Sia object +type Object struct { + fs *Fs + remote string + modTime time.Time + size int64 +} + +// Return a string version +func (o *Object) String() string { + if o == nil { + return "" + } + return o.remote +} + +// Remote returns the remote path +func (o *Object) Remote() string { + return o.remote +} + +// ModTime is the last modified time (read-only) +func (o *Object) ModTime(ctx context.Context) time.Time { + return o.modTime +} + +// Size is the file length +func (o *Object) Size() int64 { + return o.size +} + +// Fs returns the parent Fs +func (o *Object) Fs() fs.Info { + return o.fs +} + +// Hash is not supported +func (o *Object) Hash(ctx context.Context, ty hash.Type) (string, error) { + return "", hash.ErrUnsupported +} + +// Storable returns if this object is storable +func (o *Object) Storable() bool { + return true +} + +// SetModTime is not supported +func (o *Object) SetModTime(ctx context.Context, t time.Time) error { + return fs.ErrorCantSetModTime +} + +// Open an object for read +func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) { + var resp *http.Response + opts := rest.Opts{ + Method: "GET", + Path: path.Join("/renter/stream/", o.fs.root, o.fs.opt.Enc.FromStandardPath(o.remote)), + Options: options, + } + err = o.fs.pacer.Call(func() (bool, error) { + resp, err = o.fs.srv.Call(ctx, &opts) + return o.fs.shouldRetry(resp, err) + }) + if err != nil { + return nil, err + } + return resp.Body, err +} + +// Update the object with the contents of the io.Reader +func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) { + size := src.Size() + var resp *http.Response + opts := rest.Opts{ + Method: "POST", + Path: path.Join("/renter/uploadstream/", o.fs.opt.Enc.FromStandardPath(path.Join(o.fs.root, o.remote))), + Body: in, + ContentLength: &size, + Parameters: url.Values{}, + } + opts.Parameters.Set("force", "true") + + err = o.fs.pacer.Call(func() (bool, error) { + resp, err = o.fs.srv.Call(ctx, &opts) + return o.fs.shouldRetry(resp, err) + }) + + if err == nil { + err = o.readMetaData(ctx) + } + + return err +} + +// Remove an object +func (o *Object) Remove(ctx context.Context) (err error) { + var resp *http.Response + opts := rest.Opts{ + Method: "POST", + Path: path.Join("/renter/delete/", o.fs.opt.Enc.FromStandardPath(path.Join(o.fs.root, o.remote))), + } + err = o.fs.pacer.Call(func() (bool, error) { + resp, err = o.fs.srv.Call(ctx, &opts) + return o.fs.shouldRetry(resp, err) + }) + + return err +} + +// sync the size and other metadata down for the object +func (o *Object) readMetaData(ctx context.Context) (err error) { + opts := rest.Opts{ + Method: "GET", + Path: path.Join("/renter/file/", o.fs.opt.Enc.FromStandardPath(path.Join(o.fs.root, o.remote))), + } + + var result api.FileResponse + var resp *http.Response + err = o.fs.pacer.Call(func() (bool, error) { + resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &result) + return o.fs.shouldRetry(resp, err) + }) + + if err != nil { + return err + } + + o.size = int64(result.File.Filesize) + o.modTime = result.File.ModTime + + return nil +} + +// Name of the remote (as passed into NewFs) +func (f *Fs) Name() string { + return f.name +} + +// Root of the remote (as passed into NewFs) +func (f *Fs) Root() string { + return f.root +} + +// String converts this Fs to a string +func (f *Fs) String() string { + return fmt.Sprintf("Sia %s", f.opt.APIURL) +} + +// Precision is unsupported because ModTime is not changeable +func (f *Fs) Precision() time.Duration { + return fs.ModTimeNotSupported +} + +// Hashes are not exposed anywhere +func (f *Fs) Hashes() hash.Set { + return hash.Set(hash.None) +} + +// Features for this fs +func (f *Fs) Features() *fs.Features { + return f.features +} + +// List files and directories in a directory +func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) { + dirPrefix := f.opt.Enc.FromStandardPath(path.Join(f.root, dir)) + "/" + + var result api.DirectoriesResponse + var resp *http.Response + opts := rest.Opts{ + Method: "GET", + Path: path.Join("/renter/dir/", dirPrefix) + "/", + } + + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) + return f.shouldRetry(resp, err) + }) + + if err != nil { + return nil, err + } + + for _, directory := range result.Directories { + if directory.SiaPath+"/" == dirPrefix { + continue + } + + d := fs.NewDir(f.opt.Enc.ToStandardPath(strings.TrimPrefix(directory.SiaPath, f.opt.Enc.FromStandardPath(f.root)+"/")), directory.MostRecentModTime) + entries = append(entries, d) + } + + for _, file := range result.Files { + o := &Object{fs: f, + remote: f.opt.Enc.ToStandardPath(strings.TrimPrefix(file.SiaPath, f.opt.Enc.FromStandardPath(f.root)+"/")), + modTime: file.ModTime, + size: int64(file.Filesize)} + entries = append(entries, o) + } + + return entries, nil +} + +// NewObject finds the Object at remote. If it can't be found +// it returns the error fs.ErrorObjectNotFound. +func (f *Fs) NewObject(ctx context.Context, remote string) (o fs.Object, err error) { + obj := &Object{ + fs: f, + remote: remote, + } + err = obj.readMetaData(ctx) + if err != nil { + return nil, err + } + + return obj, nil +} + +// Put the object into the remote siad via uploadstream +func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + o := &Object{ + fs: f, + remote: src.Remote(), + modTime: src.ModTime(ctx), + size: src.Size(), + } + + return o, o.Update(ctx, in, src, options...) +} + +// PutStream the object into the remote siad via uploadstream +func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + return f.Put(ctx, in, src, options...) +} + +// Mkdir creates a directory +func (f *Fs) Mkdir(ctx context.Context, dir string) (err error) { + var resp *http.Response + opts := rest.Opts{ + Method: "POST", + Path: path.Join("/renter/dir/", f.opt.Enc.FromStandardPath(path.Join(f.root, dir))), + Parameters: url.Values{}, + } + opts.Parameters.Set("action", "create") + + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.Call(ctx, &opts) + return f.shouldRetry(resp, err) + }) + + if err == fs.ErrorDirExists { + err = nil + } + + return err +} + +// Rmdir removes a directory +func (f *Fs) Rmdir(ctx context.Context, dir string) (err error) { + var resp *http.Response + opts := rest.Opts{ + Method: "GET", + Path: path.Join("/renter/dir/", f.opt.Enc.FromStandardPath(path.Join(f.root, dir))), + } + + var result api.DirectoriesResponse + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) + return f.shouldRetry(resp, err) + }) + + if len(result.Directories) == 0 { + return fs.ErrorDirNotFound + } else if len(result.Files) > 0 || len(result.Directories) > 1 { + return fs.ErrorDirectoryNotEmpty + } + + opts = rest.Opts{ + Method: "POST", + Path: path.Join("/renter/dir/", f.opt.Enc.FromStandardPath(path.Join(f.root, dir))), + Parameters: url.Values{}, + } + opts.Parameters.Set("action", "delete") + + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.Call(ctx, &opts) + return f.shouldRetry(resp, err) + }) + + return err +} + +// NewFs constructs an Fs from the path +func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { + // Parse config into Options struct + opt := new(Options) + err := configstruct.Set(m, opt) + if err != nil { + return nil, err + } + + if strings.HasSuffix(opt.APIURL, "/") { + opt.APIURL = strings.TrimSuffix(opt.APIURL, "/") + } + + // Parse the endpoint + u, err := url.Parse(opt.APIURL) + if err != nil { + return nil, err + } + + rootIsDir := strings.HasSuffix(root, "/") + root = strings.Trim(root, "/") + + config := fs.Config + if opt.UserAgent != "" { + config.UserAgent = opt.UserAgent + } + + f := &Fs{ + name: name, + opt: *opt, + srv: rest.NewClient(fshttp.NewClient(config)).SetErrorHandler(errorHandler).SetRoot(u.String()), + root: root, + pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))), + } + + f.features = (&fs.Features{ + CanHaveEmptyDirectories: true, + }).Fill(f) + + if opt.APIPassword != "" { + opt.APIPassword, err = obscure.Reveal(opt.APIPassword) + if err != nil { + return nil, errors.Wrap(err, "couldn't decrypt API password") + } + f.srv.SetUserPass("", opt.APIPassword) + } + + if root != "" && !rootIsDir { + // Check to see if the root actually an existing file + remote := path.Base(root) + f.root = path.Dir(root) + if f.root == "." { + f.root = "" + } + ctx := context.Background() + _, err := f.NewObject(ctx, remote) + if err != nil { + if errors.Cause(err) == fs.ErrorObjectNotFound || errors.Cause(err) == fs.ErrorNotAFile { + // File doesn't exist so return old f + f.root = root + return f, nil + } + return nil, err + } + // return an error with an fs which points to the parent + return f, fs.ErrorIsFile + } + + return f, nil +} + +// Decode errors into meaningful ones, sadly this is using +// string matching since siad doesn't expose meaningful error codes +func errorHandler(resp *http.Response) error { + body, err := rest.ReadBody(resp) + if err != nil { + return errors.Wrap(err, "error when trying to read error body") + } + // Decode error response + errResponse := new(api.Error) + err = json.Unmarshal(body, &errResponse) + if err != nil { + // set the Message to be the body if we can't parse the JSON + errResponse.Message = strings.TrimSpace(string(body)) + } + errResponse.Status = resp.Status + errResponse.StatusCode = resp.StatusCode + + if errResponse.StatusCode == 400 && errResponse.Message == "no file known with that path" { + return fs.ErrorObjectNotFound + } else if errResponse.StatusCode == 500 && errResponse.Message == "failed to create directory: a siadir already exists at that location" { + return fs.ErrorDirExists + } else if errResponse.StatusCode == 500 && strings.HasSuffix(errResponse.Message, ": no such file or directory") { + return fs.ErrorDirNotFound + } + return errResponse +} + +// shouldRetry returns a boolean as to whether this resp and err +// deserve to be retried. It returns the err as a convenience +func (f *Fs) shouldRetry(resp *http.Response, err error) (bool, error) { + return fserrors.ShouldRetry(err), err +} diff --git a/backend/sia/sia_test.go b/backend/sia/sia_test.go new file mode 100644 index 000000000..96d42224a --- /dev/null +++ b/backend/sia/sia_test.go @@ -0,0 +1,18 @@ +// Test Sia filesystem interface +package sia_test + +import ( + "testing" + + "github.com/rclone/rclone/backend/sia" + + "github.com/rclone/rclone/fstest/fstests" +) + +// TestIntegration runs integration tests against the remote +func TestIntegration(t *testing.T) { + fstests.Run(t, &fstests.Opt{ + RemoteName: "TestSia:", + NilObject: (*sia.Object)(nil), + }) +} diff --git a/docs/content/sia.md b/docs/content/sia.md new file mode 100644 index 000000000..a09780a54 --- /dev/null +++ b/docs/content/sia.md @@ -0,0 +1,140 @@ +--- +title: "Sia" +description: "Rclone docs for Sia" +date: "2019-10-02" +--- + + Sia +----------------------------------------- + +Sia is the [Sia Decentralized Cloud](https://sia.tech/). + +You will need to be running a copy of Sia-UI or siad, locally or on your LAN (e.g. a NAS). Sia's HTTP API is required and typically listens on port 9980. + +Here is an example of how to make a remote called `remote`. First run: + + rclone config + +This will guide you through an interactive setup process: + +``` +No remotes found - make a new one +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n +name> remote +Type of storage to configure. +Enter a string value. Press Enter for the default (""). +Choose a number from below, or type in your own value +... +29 / Sia Decentralized Cloud + \ "sia" +30 / Transparently chunk/split large files + \ "chunker" +31 / Union merges the contents of several remotes + \ "union" +... +Storage> 29 +** See help for sia backend at: https://rclone.org/sia/ ** + +Sia HTTP API URL +Like http://127.0.0.1:9980 +Enter a string value. Press Enter for the default (""). +api_url> http://127.0.0.1:9980 +Sia API Password +siad API Password +y) Yes type in my own password +g) Generate random password +n) No leave this optional password blank +y/g/n> y +Enter the password: +password: +Confirm the password: +password: +Edit advanced config? (y/n) +y) Yes +n) No +y/n> n +Remote config +-------------------- +[remote] +type = sia +api_url = http://127.0.0.1:9980 +api_password = *** ENCRYPTED *** +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> y +Current remotes: + +Name Type +==== ==== +remote sia + +e) Edit existing remote +n) New remote +d) Delete remote +r) Rename remote +c) Copy remote +s) Set configuration password +q) Quit config +e/n/d/r/c/s/q> q +``` + +Once configured you can then use `rclone` like this, + +List directories in top level of your Sia storage + + rclone lsd remote: + +List all the files in your Sia storage + + rclone ls remote: + +To copy a local directory to an Sia directory called backup + + rclone copy /home/source remote:backup + + + +### Standard Options + +Here are the standard options specific to sia (Sia Decentralized Cloud). + +#### --sia-api-url + +Sia HTTP API URL +Like http://127.0.0.1:9980 + +- Config: api_url +- Env Var: RCLONE_SIA_API_URL +- Type: string +- Default: "" + +#### --sia-api-password + +Sia API Password +siad API Password + +- Config: api_password +- Env Var: RCLONE_SIA_API_PASSWORD +- Type: string +- Default: "" + +### Advanced Options + +Here are the advanced options specific to sia (Sia Decentralized Cloud). + +#### --sia-sia-user-agent + +Siad User Agent +Sia requires a 'Sia-Agent' user agent by default for security + +- Config: sia_user_agent +- Env Var: RCLONE_SIA_SIA_USER_AGENT +- Type: string +- Default: "Sia-Agent" + + diff --git a/fstest/test_all/config.yaml b/fstest/test_all/config.yaml index e24ad56b2..fb9383059 100644 --- a/fstest/test_all/config.yaml +++ b/fstest/test_all/config.yaml @@ -294,6 +294,9 @@ backends: - backend: "sharefile" remote: "TestSharefile:" fastlist: false + - backend: "sia" + remote: "TestSia:" + fastlist: false - backend: "mailru" remote: "TestMailru:" subdir: false