From ff0b8e10af347e9535cc15d0d23368bc4ac578bf Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Sat, 12 Jan 2019 16:46:45 +0000 Subject: [PATCH] s3: Support Alibaba Cloud (Aliyun) OSS The existing s3 backend passed all integration tests with OSS provided `force_path_style = false`. This makes sure that is so and adds documentation and configuration for OSS. Thanks to @luolibin for their work on the OSS backend which we ended up not needing. Fixes #1641 Fixes #1237 --- CONTRIBUTING.md | 1 - README.md | 1 + backend/s3/s3.go | 101 ++++++++++++++++++++-- docs/content/about.md | 1 + docs/content/s3.md | 191 +++++++++++++++++++++++++++++------------- 5 files changed, 230 insertions(+), 65 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dc4275046..f040f6a72 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -372,4 +372,3 @@ Add your fs to the docs - you'll need to pick an icon for it from [fontawesome]( * `docs/content/about.md` - front page of rclone.org * `docs/layouts/chrome/navbar.html` - add it to the website navigation * `bin/make_manual.py` - add the page to the `docs` constant - * `cmd/cmd.go` - the main help for rclone diff --git a/README.md b/README.md index c4bbf55e9..e61f0fb4e 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ Rclone *("rsync for cloud storage")* is a command line program to sync files and ## Storage providers + * Alibaba Cloud (Aliyun) Object Storage System (OSS) [:page_facing_up:](https://rclone.org/s3/#alibaba-oss) * Amazon Drive [:page_facing_up:](https://rclone.org/amazonclouddrive/) ([See note](https://rclone.org/amazonclouddrive/#status)) * Amazon S3 [:page_facing_up:](https://rclone.org/s3/) * Backblaze B2 [:page_facing_up:](https://rclone.org/b2/) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 49d98c395..9d1c4f3c5 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -53,7 +53,7 @@ import ( func init() { fs.Register(&fs.RegInfo{ Name: "s3", - Description: "Amazon S3 Compliant Storage Providers (AWS, Ceph, Dreamhost, IBM COS, Minio)", + Description: "Amazon S3 Compliant Storage Provider (AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, etc)", NewFs: NewFs, Options: []fs.Option{{ Name: fs.ConfigProvider, @@ -61,6 +61,9 @@ func init() { Examples: []fs.OptionExample{{ Value: "AWS", Help: "Amazon Web Services (AWS) S3", + }, { + Value: "Alibaba", + Help: "Alibaba Cloud Object Storage System (OSS) formerly Aliyun", }, { Value: "Ceph", Help: "Ceph Object Storage", @@ -76,6 +79,9 @@ func init() { }, { Value: "Minio", Help: "Minio Object Storage", + }, { + Value: "Netease", + Help: "Netease Object Storage (NOS)", }, { Value: "Wasabi", Help: "Wasabi Object Storage", @@ -150,7 +156,7 @@ func init() { }, { Name: "region", Help: "Region to connect to.\nLeave blank if you are using an S3 clone and you don't have a region.", - Provider: "!AWS", + Provider: "!AWS,Alibaba", Examples: []fs.OptionExample{{ Value: "", Help: "Use this if unsure. Will use v4 signatures and an empty region.", @@ -269,10 +275,73 @@ func init() { Value: "s3.tor01.objectstorage.service.networklayer.com", Help: "Toronto Single Site Private Endpoint", }}, + }, { + // oss endpoints: https://help.aliyun.com/document_detail/31837.html + Name: "endpoint", + Help: "Endpoint for OSS API.", + Provider: "Alibaba", + Examples: []fs.OptionExample{{ + Value: "oss-cn-hangzhou.aliyuncs.com", + Help: "East China 1 (Hangzhou)", + }, { + Value: "oss-cn-shanghai.aliyuncs.com", + Help: "East China 2 (Shanghai)", + }, { + Value: "oss-cn-qingdao.aliyuncs.com", + Help: "North China 1 (Qingdao)", + }, { + Value: "oss-cn-beijing.aliyuncs.com", + Help: "North China 2 (Beijing)", + }, { + Value: "oss-cn-zhangjiakou.aliyuncs.com", + Help: "North China 3 (Zhangjiakou)", + }, { + Value: "oss-cn-huhehaote.aliyuncs.com", + Help: "North China 5 (Huhehaote)", + }, { + Value: "oss-cn-shenzhen.aliyuncs.com", + Help: "South China 1 (Shenzhen)", + }, { + Value: "oss-cn-hongkong.aliyuncs.com", + Help: "Hong Kong (Hong Kong)", + }, { + Value: "oss-us-west-1.aliyuncs.com", + Help: "US West 1 (Silicon Valley)", + }, { + Value: "oss-us-east-1.aliyuncs.com", + Help: "US East 1 (Virginia)", + }, { + Value: "oss-ap-southeast-1.aliyuncs.com", + Help: "Southeast Asia Southeast 1 (Singapore)", + }, { + Value: "oss-ap-southeast-2.aliyuncs.com", + Help: "Asia Pacific Southeast 2 (Sydney)", + }, { + Value: "oss-ap-southeast-3.aliyuncs.com", + Help: "Southeast Asia Southeast 3 (Kuala Lumpur)", + }, { + Value: "oss-ap-southeast-5.aliyuncs.com", + Help: "Asia Pacific Southeast 5 (Jakarta)", + }, { + Value: "oss-ap-northeast-1.aliyuncs.com", + Help: "Asia Pacific Northeast 1 (Japan)", + }, { + Value: "oss-ap-south-1.aliyuncs.com", + Help: "Asia Pacific South 1 (Mumbai)", + }, { + Value: "oss-eu-central-1.aliyuncs.com", + Help: "Central Europe 1 (Frankfurt)", + }, { + Value: "oss-eu-west-1.aliyuncs.com", + Help: "West Europe (London)", + }, { + Value: "oss-me-east-1.aliyuncs.com", + Help: "Middle East 1 (Dubai)", + }}, }, { Name: "endpoint", Help: "Endpoint for S3 API.\nRequired when using an S3 clone.", - Provider: "!AWS,IBMCOS", + Provider: "!AWS,IBMCOS,Alibaba", Examples: []fs.OptionExample{{ Value: "objects-us-west-1.dream.io", Help: "Dream Objects endpoint", @@ -449,7 +518,7 @@ func init() { }, { Name: "location_constraint", Help: "Location constraint - must be set to match the Region.\nLeave blank if not sure. Used when creating buckets only.", - Provider: "!AWS,IBMCOS", + Provider: "!AWS,IBMCOS,Alibaba", }, { Name: "acl", Help: `Canned ACL used when creating buckets and storing or copying objects. @@ -547,6 +616,20 @@ doesn't copy the ACL from the source but rather writes a fresh one.`, Value: "GLACIER", Help: "Glacier storage class", }}, + }, { + Name: "storage_class", + Help: "The storage class to use when storing new objects in OSS.", + Provider: "Alibaba", + Examples: []fs.OptionExample{{ + Value: "Standard", + Help: "Standard storage class", + }, { + Value: "Archive", + Help: "Archive storage mode.", + }, { + Value: "IA", + Help: "Infrequent access storage mode.", + }}, }, { Name: "upload_cutoff", Help: `Cutoff for switching to chunked upload @@ -714,7 +797,7 @@ func (f *Fs) Features() *fs.Features { // retryErrorCodes is a slice of error codes that we will retry // See: https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html var retryErrorCodes = []int{ - 409, // Conflict - various states that could be resolved on a retry + // 409, // Conflict - various states that could be resolved on a retry 503, // Service Unavailable/Slow Down - "Reduce your request rate" } @@ -722,14 +805,13 @@ var retryErrorCodes = []int{ // as it should notice closed connections and timeouts which are the most likely // sort of failure modes func shouldRetry(err error) (bool, error) { - // If this is an awserr object, try and extract more useful information to determine if we should retry if awsError, ok := err.(awserr.Error); ok { // Simple case, check the original embedded error in case it's generically retriable if fserrors.ShouldRetry(awsError.OrigErr()) { return true, err } - //Failing that, if it's a RequestFailure it's probably got an http status code we can check + // Failing that, if it's a RequestFailure it's probably got an http status code we can check if reqErr, ok := err.(awserr.RequestFailure); ok { for _, e := range retryErrorCodes { if reqErr.StatusCode() == e { @@ -738,7 +820,7 @@ func shouldRetry(err error) (bool, error) { } } } - //Ok, not an awserr, check for generic failure conditions + // Ok, not an awserr, check for generic failure conditions return fserrors.ShouldRetry(err), err } @@ -815,6 +897,9 @@ func s3Connection(opt *Options) (*s3.S3, *session.Session, error) { if opt.Region == "" { opt.Region = "us-east-1" } + if opt.Provider == "Alibaba" || opt.Provider == "Netease" { + opt.ForcePathStyle = false + } awsConfig := aws.NewConfig(). WithRegion(opt.Region). WithMaxRetries(maxRetries). diff --git a/docs/content/about.md b/docs/content/about.md index 05d4acdf8..1e0bfb45f 100644 --- a/docs/content/about.md +++ b/docs/content/about.md @@ -13,6 +13,7 @@ Rclone Rclone is a command line program to sync files and directories to and from: +* {{< provider name="Alibaba Cloud (Aliyun) Object Storage System (OSS)" home="https://www.alibabacloud.com/product/oss/" config="/s3/#alibaba-oss" >}} * {{< provider name="Amazon Drive" home="https://www.amazon.com/clouddrive" config="/amazonclouddrive/" >}} ([See note](/amazonclouddrive/#status)) * {{< provider name="Amazon S3" home="https://aws.amazon.com/s3/" config="/s3/" >}} * {{< provider name="Backblaze B2" home="https://www.backblaze.com/b2/cloud-storage.html" config="/b2/" >}} diff --git a/docs/content/s3.md b/docs/content/s3.md index 4110edcb4..9dd726ac4 100644 --- a/docs/content/s3.md +++ b/docs/content/s3.md @@ -10,6 +10,7 @@ date: "2016-07-11" The S3 backend can be used with a number of different providers: * {{< provider name="AWS S3" home="https://aws.amazon.com/s3/" config="/s3/#amazon-s3" >}} +* {{< provider name="Alibaba Cloud (Aliyun) Object Storage System (OSS)" home="https://www.alibabacloud.com/product/oss/" config="/s3/#alibaba-oss" >}} * {{< provider name="Ceph" home="http://ceph.com/" config="/s3/#ceph" >}} * {{< provider name="DigitalOcean Spaces" home="https://www.digitalocean.com/products/object-storage/" config="/s3/#digitalocean-spaces" >}} * {{< provider name="Dreamhost" home="https://www.dreamhost.com/cloud/storage/" config="/s3/#dreamhost" >}} @@ -400,7 +401,7 @@ the object(s) in question before using rclone. ### Standard Options -Here are the standard options specific to s3 (Amazon S3 Compliant Storage Providers (AWS, Ceph, Dreamhost, IBM COS, Minio)). +Here are the standard options specific to s3 (Amazon S3 Compliant Storage Provider (AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, etc)). #### --s3-provider @@ -413,6 +414,8 @@ Choose your S3 provider. - Examples: - "AWS" - Amazon Web Services (AWS) S3 + - "Alibaba" + - Alibaba Cloud Object Storage System (OSS) formerly Aliyun - "Ceph" - Ceph Object Storage - "DigitalOcean" @@ -423,6 +426,8 @@ Choose your S3 provider. - IBM COS S3 - "Minio" - Minio Object Storage + - "Netease" + - Netease Object Storage (NOS) - "Wasabi" - Wasabi Object Storage - "Other" @@ -622,6 +627,54 @@ Specify if using an IBM COS On Premise. #### --s3-endpoint +Endpoint for OSS API. + +- Config: endpoint +- Env Var: RCLONE_S3_ENDPOINT +- Type: string +- Default: "" +- Examples: + - "oss-cn-hangzhou.aliyuncs.com" + - East China 1 (Hangzhou) + - "oss-cn-shanghai.aliyuncs.com" + - East China 2 (Shanghai) + - "oss-cn-qingdao.aliyuncs.com" + - North China 1 (Qingdao) + - "oss-cn-beijing.aliyuncs.com" + - North China 2 (Beijing) + - "oss-cn-zhangjiakou.aliyuncs.com" + - North China 3 (Zhangjiakou) + - "oss-cn-huhehaote.aliyuncs.com" + - North China 5 (Huhehaote) + - "oss-cn-shenzhen.aliyuncs.com" + - South China 1 (Shenzhen) + - "oss-cn-hongkong.aliyuncs.com" + - Hong Kong (Hong Kong) + - "oss-us-west-1.aliyuncs.com" + - US West 1 (Silicon Valley) + - "oss-us-east-1.aliyuncs.com" + - US East 1 (Virginia) + - "oss-ap-southeast-1.aliyuncs.com" + - Southeast Asia Southeast 1 (Singapore) + - "oss-ap-southeast-2.aliyuncs.com" + - Asia Pacific Southeast 2 (Sydney) + - "oss-ap-southeast-3.aliyuncs.com" + - Southeast Asia Southeast 3 (Kuala Lumpur) + - "oss-ap-southeast-5.aliyuncs.com" + - Asia Pacific Southeast 5 (Jakarta) + - "oss-ap-northeast-1.aliyuncs.com" + - Asia Pacific Northeast 1 (Japan) + - "oss-ap-south-1.aliyuncs.com" + - Asia Pacific South 1 (Mumbai) + - "oss-eu-central-1.aliyuncs.com" + - Central Europe 1 (Frankfurt) + - "oss-eu-west-1.aliyuncs.com" + - West Europe (London) + - "oss-me-east-1.aliyuncs.com" + - Middle East 1 (Dubai) + +#### --s3-endpoint + Endpoint for S3 API. Required when using an S3 clone. @@ -855,11 +908,27 @@ The storage class to use when storing new objects in S3. - "ONEZONE_IA" - One Zone Infrequent Access storage class - "GLACIER" - - GLACIER storage class + - Glacier storage class + +#### --s3-storage-class + +The storage class to use when storing new objects in OSS. + +- Config: storage_class +- Env Var: RCLONE_S3_STORAGE_CLASS +- Type: string +- Default: "" +- Examples: + - "Standard" + - Standard storage class + - "Archive" + - Archive storage mode. + - "IA" + - Infrequent access storage mode. ### Advanced Options -Here are the advanced options specific to s3 (Amazon S3 Compliant Storage Providers (AWS, Ceph, Dreamhost, IBM COS, Minio)). +Here are the advanced options specific to s3 (Amazon S3 Compliant Storage Provider (AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, etc)). #### --s3-upload-cutoff @@ -1458,30 +1527,41 @@ server_side_encryption = storage_class = ``` -### Aliyun OSS / Netease NOS ### +### Alibaba OSS {#alibaba-oss} -This describes how to set up Aliyun OSS - Netease NOS is the same -except for different endpoints. +Here is an example of making an [Alibaba Cloud (Aliyun) OSS](https://www.alibabacloud.com/product/oss/) +configuration. First run: -Note this is a pretty standard S3 setup, except for the setting of -`force_path_style = false` in the advanced config. + rclone config + +This will guide you through an interactive setup process. ``` -# rclone config -e/n/d/r/c/s/q> n +No remotes found - make a new one +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n name> oss Type of storage to configure. Enter a string value. Press Enter for the default (""). Choose a number from below, or type in your own value - 3 / Amazon S3 Compliant Storage Providers (AWS, Ceph, Dreamhost, IBM COS, Minio) +[snip] + 4 / Amazon S3 Compliant Storage Provider (AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, etc) \ "s3" +[snip] Storage> s3 Choose your S3 provider. Enter a string value. Press Enter for the default (""). Choose a number from below, or type in your own value - 8 / Any other S3 compatible provider - \ "Other" -provider> other + 1 / Amazon Web Services (AWS) S3 + \ "AWS" + 2 / Alibaba Cloud Object Storage System (OSS) formerly Aliyun + \ "Alibaba" + 3 / Ceph Object Storage + \ "Ceph" +[snip] +provider> Alibaba Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). Only applies if access_key_id and secret_access_key is blank. Enter a boolean value (true or false). Press Enter for the default ("false"). @@ -1494,70 +1574,69 @@ env_auth> 1 AWS Access Key ID. Leave blank for anonymous access or runtime credentials. Enter a string value. Press Enter for the default (""). -access_key_id> xxxxxxxxxxxx +access_key_id> accesskeyid AWS Secret Access Key (password) Leave blank for anonymous access or runtime credentials. Enter a string value. Press Enter for the default (""). -secret_access_key> xxxxxxxxxxxxxxxxx -Region to connect to. -Leave blank if you are using an S3 clone and you don't have a region. +secret_access_key> secretaccesskey +Endpoint for OSS API. Enter a string value. Press Enter for the default (""). Choose a number from below, or type in your own value - 1 / Use this if unsure. Will use v4 signatures and an empty region. - \ "" - 2 / Use this only if v4 signatures don't work, eg pre Jewel/v10 CEPH. - \ "other-v2-signature" -region> 1 -Endpoint for S3 API. -Required when using an S3 clone. -Enter a string value. Press Enter for the default (""). -Choose a number from below, or type in your own value -endpoint> oss-cn-shenzhen.aliyuncs.com -Location constraint - must be set to match the Region. -Leave blank if not sure. Used when creating buckets only. -Enter a string value. Press Enter for the default (""). -location_constraint> -Canned ACL used when creating buckets and/or storing objects in S3. -For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl + 1 / East China 1 (Hangzhou) + \ "oss-cn-hangzhou.aliyuncs.com" + 2 / East China 2 (Shanghai) + \ "oss-cn-shanghai.aliyuncs.com" + 3 / North China 1 (Qingdao) + \ "oss-cn-qingdao.aliyuncs.com" +[snip] +endpoint> 1 +Canned ACL used when creating buckets and storing or copying objects. + +Note that this ACL is applied when server side copying objects as S3 +doesn't copy the ACL from the source but rather writes a fresh one. Enter a string value. Press Enter for the default (""). Choose a number from below, or type in your own value 1 / Owner gets FULL_CONTROL. No one else has access rights (default). \ "private" + 2 / Owner gets FULL_CONTROL. The AllUsers group gets READ access. + \ "public-read" + / Owner gets FULL_CONTROL. The AllUsers group gets READ and WRITE access. +[snip] acl> 1 +The storage class to use when storing new objects in OSS. +Enter a string value. Press Enter for the default (""). +Choose a number from below, or type in your own value + 1 / Standard storage class + \ "Standard" + 2 / Archive storage mode. + \ "Archive" + 3 / Infrequent access storage mode. + \ "IA" +storage_class> 1 Edit advanced config? (y/n) y) Yes n) No -y/n> y -Chunk size to use for uploading -Enter a size with suffix k,M,G,T. Press Enter for the default ("5M"). -chunk_size> -Don't store MD5 checksum with object metadata -Enter a boolean value (true or false). Press Enter for the default ("false"). -disable_checksum> -An AWS session token -Enter a string value. Press Enter for the default (""). -session_token> -Concurrency for multipart uploads. -Enter a signed integer. Press Enter for the default ("2"). -upload_concurrency> -If true use path style access if false use virtual hosted style. -Some providers (eg Aliyun OSS or Netease COS) require this. -Enter a boolean value (true or false). Press Enter for the default ("true"). -force_path_style> false +y/n> n Remote config -------------------- [oss] type = s3 -provider = Other +provider = Alibaba env_auth = false -access_key_id = xxxxxxxxx -secret_access_key = xxxxxxxxxxxxx -endpoint = oss-cn-shenzhen.aliyuncs.com +access_key_id = accesskeyid +secret_access_key = secretaccesskey +endpoint = oss-cn-hangzhou.aliyuncs.com acl = private -force_path_style = false +storage_class = Standard -------------------- y) Yes this is OK e) Edit this remote d) Delete this remote y/e/d> y ``` + +### Netease NOS ### + +For Netease NOS configure as per the configurator `rclone config` +setting the provider `Netease`. This will automatically set +`force_path_style = false` which is necessary for it to run properly.