s3: add specific provider for Storj Shared gateways
- unsupported features (Copy) are turned off for Storj - enable urlEncodedListing for Storj provider - set chunksize to 64Mb
This commit is contained in:
Márton Elek
Nick Craig-Wood
parent
06ffd4882d
commit
25ea04f1db
|
|
@ -102,6 +102,9 @@ func init() {
|
|||
}, {
|
||||
Value: "StackPath",
|
||||
Help: "StackPath Object Storage",
|
||||
}, {
|
||||
Value: "Storj",
|
||||
Help: "Storj (S3 Compatible Gateway)",
|
||||
}, {
|
||||
Value: "TencentCOS",
|
||||
Help: "Tencent Cloud Object Storage (COS)",
|
||||
|
|
@ -288,7 +291,7 @@ func init() {
|
|||
}, {
|
||||
Name: "region",
|
||||
Help: "Region to connect to.\n\nLeave blank if you are using an S3 clone and you don't have a region.",
|
||||
Provider: "!AWS,Alibaba,RackCorp,Scaleway,TencentCOS",
|
||||
Provider: "!AWS,Alibaba,RackCorp,Scaleway,Storj,TencentCOS",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "",
|
||||
Help: "Use this if unsure.\nWill use v4 signatures and an empty region.",
|
||||
|
|
@ -597,6 +600,20 @@ func init() {
|
|||
Value: "s3.eu-central-1.stackpathstorage.com",
|
||||
Help: "EU Endpoint",
|
||||
}},
|
||||
}, {
|
||||
Name: "endpoint",
|
||||
Help: "Endpoint of the Shared Gateway.",
|
||||
Provider: "Storj",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "gateway.eu1.storjshare.io",
|
||||
Help: "EU1 Shared Gateway",
|
||||
}, {
|
||||
Value: "gateway.us1.storjshare.io",
|
||||
Help: "US1 Shared Gateway",
|
||||
}, {
|
||||
Value: "gateway.ap1.storjshare.io",
|
||||
Help: "Asia-Pacific Shared Gateway",
|
||||
}},
|
||||
}, {
|
||||
// cos endpoints: https://intl.cloud.tencent.com/document/product/436/6224
|
||||
Name: "endpoint",
|
||||
|
|
@ -726,7 +743,7 @@ func init() {
|
|||
}, {
|
||||
Name: "endpoint",
|
||||
Help: "Endpoint for S3 API.\n\nRequired when using an S3 clone.",
|
||||
Provider: "!AWS,IBMCOS,TencentCOS,Alibaba,Scaleway,StackPath,RackCorp",
|
||||
Provider: "!AWS,IBMCOS,TencentCOS,Alibaba,Scaleway,StackPath,Storj,RackCorp",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "objects-us-east-1.dream.io",
|
||||
Help: "Dream Objects endpoint",
|
||||
|
|
@ -1014,7 +1031,7 @@ func init() {
|
|||
}, {
|
||||
Name: "location_constraint",
|
||||
Help: "Location constraint - must be set to match the Region.\n\nLeave blank if not sure. Used when creating buckets only.",
|
||||
Provider: "!AWS,IBMCOS,Alibaba,RackCorp,Scaleway,StackPath,TencentCOS",
|
||||
Provider: "!AWS,IBMCOS,Alibaba,RackCorp,Scaleway,StackPath,Storj,TencentCOS",
|
||||
}, {
|
||||
Name: "acl",
|
||||
Help: `Canned ACL used when creating buckets and storing or copying objects.
|
||||
|
|
@ -1025,6 +1042,7 @@ For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview
|
|||
|
||||
Note that this ACL is applied when server-side copying objects as S3
|
||||
doesn't copy the ACL from the source but rather writes a fresh one.`,
|
||||
Provider: "!Storj",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "default",
|
||||
Help: "Owner gets Full_CONTROL.\nNo one else has access rights (default).",
|
||||
|
|
@ -1936,6 +1954,11 @@ func setQuirks(opt *Options) {
|
|||
listObjectsV2 = false // untested
|
||||
virtualHostStyle = false
|
||||
urlEncodeListings = false
|
||||
case "Storj":
|
||||
// Force chunk size to >= 64 MiB
|
||||
if opt.ChunkSize < 64*fs.Mebi {
|
||||
opt.ChunkSize = 64 * fs.Mebi
|
||||
}
|
||||
case "TencentCOS":
|
||||
listObjectsV2 = false // untested
|
||||
case "Wasabi":
|
||||
|
|
@ -2070,6 +2093,11 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
|||
// return an error with an fs which points to the parent
|
||||
return f, fs.ErrorIsFile
|
||||
}
|
||||
if opt.Provider == "Storj" {
|
||||
f.features.Copy = nil
|
||||
f.features.SetTier = false
|
||||
f.features.GetTier = false
|
||||
}
|
||||
// f.listMultipartUploads()
|
||||
return f, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ The S3 backend can be used with a number of different providers:
|
|||
{{< provider name="Scaleway" home="https://www.scaleway.com/en/object-storage/" config="/s3/#scaleway" >}}
|
||||
{{< provider name="SeaweedFS" home="https://github.com/chrislusf/seaweedfs/" config="/s3/#seaweedfs" >}}
|
||||
{{< provider name="StackPath" home="https://www.stackpath.com/products/object-storage/" config="/s3/#stackpath" >}}
|
||||
{{< provider name="Storj" home="https://storj.io/" config="/s3/#storj" >}}
|
||||
{{< provider name="Tencent Cloud Object Storage (COS)" home="https://intl.cloud.tencent.com/product/cos" config="/s3/#tencent-cos" >}}
|
||||
{{< provider name="Wasabi" home="https://wasabi.com/" config="/s3/#wasabi" end="true" >}}
|
||||
{{< /provider_list >}}
|
||||
|
|
@ -2821,6 +2822,83 @@ For Netease NOS configure as per the configurator `rclone config`
|
|||
setting the provider `Netease`. This will automatically set
|
||||
`force_path_style = false` which is necessary for it to run properly.
|
||||
|
||||
### Storj
|
||||
|
||||
Storj is a decentralized cloud storage which can be used through its
|
||||
native protocol or an S3 compatible gateway.
|
||||
|
||||
The S3 compatible gateway is configured using `rclone config` with a
|
||||
type of `s3` and with a provider name of `Storj`. Here is an example
|
||||
run of the configurator.
|
||||
|
||||
```
|
||||
Type of storage to configure.
|
||||
Storage> s3
|
||||
Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).
|
||||
Only applies if access_key_id and secret_access_key is blank.
|
||||
Choose a number from below, or type in your own boolean value (true or false).
|
||||
Press Enter for the default (false).
|
||||
1 / Enter AWS credentials in the next step.
|
||||
\ (false)
|
||||
2 / Get AWS credentials from the environment (env vars or IAM).
|
||||
\ (true)
|
||||
env_auth> 1
|
||||
Option access_key_id.
|
||||
AWS Access Key ID.
|
||||
Leave blank for anonymous access or runtime credentials.
|
||||
Enter a value. Press Enter to leave empty.
|
||||
access_key_id> XXXX (as shown when creating the access grant)
|
||||
Option secret_access_key.
|
||||
AWS Secret Access Key (password).
|
||||
Leave blank for anonymous access or runtime credentials.
|
||||
Enter a value. Press Enter to leave empty.
|
||||
secret_access_key> XXXX (as shown when creating the access grant)
|
||||
Option endpoint.
|
||||
Endpoint of the Shared Gateway.
|
||||
Choose a number from below, or type in your own value.
|
||||
Press Enter to leave empty.
|
||||
1 / EU1 Shared Gateway
|
||||
\ (gateway.eu1.storjshare.io)
|
||||
2 / US1 Shared Gateway
|
||||
\ (gateway.us1.storjshare.io)
|
||||
3 / Asia-Pacific Shared Gateway
|
||||
\ (gateway.ap1.storjshare.io)
|
||||
endpoint> 1 (as shown when creating the access grant)
|
||||
Edit advanced config?
|
||||
y) Yes
|
||||
n) No (default)
|
||||
y/n> n
|
||||
```
|
||||
|
||||
Note that s3 credentials are generated when you [create an access
|
||||
grant](https://docs.storj.io/dcs/api-reference/s3-compatible-gateway#usage).
|
||||
|
||||
|
||||
#### Backend quirks
|
||||
|
||||
- `--chunk-size` is forced to be 64 MiB or greater. This will use more
|
||||
memory than the default of 5 MiB.
|
||||
- Server side copy is disabled as it isn't currently supported in the
|
||||
gateway.
|
||||
- GetTier and SetTier are not supported.
|
||||
|
||||
#### Comparison with the native protocol
|
||||
|
||||
Use the [the native protocol](/tardigrade) to take advantage of
|
||||
client-side encryption as well as to achieve the best possible
|
||||
download performance. Uploads will be erasure-coded locally, thus a
|
||||
1gb upload will result in 2.68gb of data being uploaded to storage
|
||||
nodes across the network.
|
||||
|
||||
Use this backend and the S3 compatible Hosted Gateway to increase
|
||||
upload performance and reduce the load on your systems and network.
|
||||
Uploads will be encrypted and erasure-coded server-side, thus a 1GB
|
||||
upload will result in only in 1GB of data being uploaded to storage
|
||||
nodes across the network.
|
||||
|
||||
For more detailed comparison please check the documentation of the
|
||||
[tardigrade](/tardigrade) backend.
|
||||
|
||||
## Limitations
|
||||
|
||||
`rclone about` is not supported by the S3 backend. Backends without
|
||||
|
|
@ -2830,4 +2908,3 @@ remote.
|
|||
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
|
||||
See [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue