diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 2ca8a8f89..55b49503a 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -102,6 +102,9 @@ func init() { }, { Value: "StackPath", Help: "StackPath Object Storage", + }, { + Value: "Storj", + Help: "Storj (S3 Compatible Gateway)", }, { Value: "TencentCOS", Help: "Tencent Cloud Object Storage (COS)", @@ -288,7 +291,7 @@ func init() { }, { Name: "region", Help: "Region to connect to.\n\nLeave blank if you are using an S3 clone and you don't have a region.", - Provider: "!AWS,Alibaba,RackCorp,Scaleway,TencentCOS", + Provider: "!AWS,Alibaba,RackCorp,Scaleway,Storj,TencentCOS", Examples: []fs.OptionExample{{ Value: "", Help: "Use this if unsure.\nWill use v4 signatures and an empty region.", @@ -597,6 +600,20 @@ func init() { Value: "s3.eu-central-1.stackpathstorage.com", Help: "EU Endpoint", }}, + }, { + Name: "endpoint", + Help: "Endpoint of the Shared Gateway.", + Provider: "Storj", + Examples: []fs.OptionExample{{ + Value: "gateway.eu1.storjshare.io", + Help: "EU1 Shared Gateway", + }, { + Value: "gateway.us1.storjshare.io", + Help: "US1 Shared Gateway", + }, { + Value: "gateway.ap1.storjshare.io", + Help: "Asia-Pacific Shared Gateway", + }}, }, { // cos endpoints: https://intl.cloud.tencent.com/document/product/436/6224 Name: "endpoint", @@ -726,7 +743,7 @@ func init() { }, { Name: "endpoint", Help: "Endpoint for S3 API.\n\nRequired when using an S3 clone.", - Provider: "!AWS,IBMCOS,TencentCOS,Alibaba,Scaleway,StackPath,RackCorp", + Provider: "!AWS,IBMCOS,TencentCOS,Alibaba,Scaleway,StackPath,Storj,RackCorp", Examples: []fs.OptionExample{{ Value: "objects-us-east-1.dream.io", Help: "Dream Objects endpoint", @@ -1014,7 +1031,7 @@ func init() { }, { Name: "location_constraint", Help: "Location constraint - must be set to match the Region.\n\nLeave blank if not sure. Used when creating buckets only.", - Provider: "!AWS,IBMCOS,Alibaba,RackCorp,Scaleway,StackPath,TencentCOS", + Provider: "!AWS,IBMCOS,Alibaba,RackCorp,Scaleway,StackPath,Storj,TencentCOS", }, { Name: "acl", Help: `Canned ACL used when creating buckets and storing or copying objects. @@ -1025,6 +1042,7 @@ For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview Note that this ACL is applied when server-side copying objects as S3 doesn't copy the ACL from the source but rather writes a fresh one.`, + Provider: "!Storj", Examples: []fs.OptionExample{{ Value: "default", Help: "Owner gets Full_CONTROL.\nNo one else has access rights (default).", @@ -1936,6 +1954,11 @@ func setQuirks(opt *Options) { listObjectsV2 = false // untested virtualHostStyle = false urlEncodeListings = false + case "Storj": + // Force chunk size to >= 64 MiB + if opt.ChunkSize < 64*fs.Mebi { + opt.ChunkSize = 64 * fs.Mebi + } case "TencentCOS": listObjectsV2 = false // untested case "Wasabi": @@ -2070,6 +2093,11 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e // return an error with an fs which points to the parent return f, fs.ErrorIsFile } + if opt.Provider == "Storj" { + f.features.Copy = nil + f.features.SetTier = false + f.features.GetTier = false + } // f.listMultipartUploads() return f, nil } diff --git a/docs/content/s3.md b/docs/content/s3.md index 67f940117..ccb63c3f5 100644 --- a/docs/content/s3.md +++ b/docs/content/s3.md @@ -19,6 +19,7 @@ The S3 backend can be used with a number of different providers: {{< provider name="Scaleway" home="https://www.scaleway.com/en/object-storage/" config="/s3/#scaleway" >}} {{< provider name="SeaweedFS" home="https://github.com/chrislusf/seaweedfs/" config="/s3/#seaweedfs" >}} {{< provider name="StackPath" home="https://www.stackpath.com/products/object-storage/" config="/s3/#stackpath" >}} +{{< provider name="Storj" home="https://storj.io/" config="/s3/#storj" >}} {{< provider name="Tencent Cloud Object Storage (COS)" home="https://intl.cloud.tencent.com/product/cos" config="/s3/#tencent-cos" >}} {{< provider name="Wasabi" home="https://wasabi.com/" config="/s3/#wasabi" end="true" >}} {{< /provider_list >}} @@ -2821,6 +2822,83 @@ For Netease NOS configure as per the configurator `rclone config` setting the provider `Netease`. This will automatically set `force_path_style = false` which is necessary for it to run properly. +### Storj + +Storj is a decentralized cloud storage which can be used through its +native protocol or an S3 compatible gateway. + +The S3 compatible gateway is configured using `rclone config` with a +type of `s3` and with a provider name of `Storj`. Here is an example +run of the configurator. + +``` +Type of storage to configure. +Storage> s3 +Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). +Only applies if access_key_id and secret_access_key is blank. +Choose a number from below, or type in your own boolean value (true or false). +Press Enter for the default (false). + 1 / Enter AWS credentials in the next step. + \ (false) + 2 / Get AWS credentials from the environment (env vars or IAM). + \ (true) +env_auth> 1 +Option access_key_id. +AWS Access Key ID. +Leave blank for anonymous access or runtime credentials. +Enter a value. Press Enter to leave empty. +access_key_id> XXXX (as shown when creating the access grant) +Option secret_access_key. +AWS Secret Access Key (password). +Leave blank for anonymous access or runtime credentials. +Enter a value. Press Enter to leave empty. +secret_access_key> XXXX (as shown when creating the access grant) +Option endpoint. +Endpoint of the Shared Gateway. +Choose a number from below, or type in your own value. +Press Enter to leave empty. + 1 / EU1 Shared Gateway + \ (gateway.eu1.storjshare.io) + 2 / US1 Shared Gateway + \ (gateway.us1.storjshare.io) + 3 / Asia-Pacific Shared Gateway + \ (gateway.ap1.storjshare.io) +endpoint> 1 (as shown when creating the access grant) +Edit advanced config? +y) Yes +n) No (default) +y/n> n +``` + +Note that s3 credentials are generated when you [create an access +grant](https://docs.storj.io/dcs/api-reference/s3-compatible-gateway#usage). + + +#### Backend quirks + +- `--chunk-size` is forced to be 64 MiB or greater. This will use more + memory than the default of 5 MiB. +- Server side copy is disabled as it isn't currently supported in the + gateway. +- GetTier and SetTier are not supported. + +#### Comparison with the native protocol + +Use the [the native protocol](/tardigrade) to take advantage of +client-side encryption as well as to achieve the best possible +download performance. Uploads will be erasure-coded locally, thus a +1gb upload will result in 2.68gb of data being uploaded to storage +nodes across the network. + +Use this backend and the S3 compatible Hosted Gateway to increase +upload performance and reduce the load on your systems and network. +Uploads will be encrypted and erasure-coded server-side, thus a 1GB +upload will result in only in 1GB of data being uploaded to storage +nodes across the network. + +For more detailed comparison please check the documentation of the +[tardigrade](/tardigrade) backend. + ## Limitations `rclone about` is not supported by the S3 backend. Backends without @@ -2830,4 +2908,3 @@ remote. See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) See [rclone about](https://rclone.org/commands/rclone_about/) -