From ce8b1cd86155c051e6a6dbe375997aed4131c017 Mon Sep 17 00:00:00 2001 From: Manoj Ghosh Date: Thu, 16 Feb 2023 12:07:34 -0800 Subject: [PATCH] oracle-object-storage: bring your own encryption keys --- backend/oracleobjectstorage/byok.go | 145 ++++++++++++++++++ backend/oracleobjectstorage/client.go | 26 +++- backend/oracleobjectstorage/copy.go | 1 + backend/oracleobjectstorage/object.go | 9 +- backend/oracleobjectstorage/options.go | 93 +++++++++-- .../oracleobjectstorage.go | 4 + docs/content/oracleobjectstorage.md | 83 ++++++++++ go.mod | 2 +- go.sum | 4 +- 9 files changed, 342 insertions(+), 25 deletions(-) create mode 100644 backend/oracleobjectstorage/byok.go diff --git a/backend/oracleobjectstorage/byok.go b/backend/oracleobjectstorage/byok.go new file mode 100644 index 000000000..099476a7d --- /dev/null +++ b/backend/oracleobjectstorage/byok.go @@ -0,0 +1,145 @@ +//go:build !plan9 && !solaris && !js +// +build !plan9,!solaris,!js + +package oracleobjectstorage + +import ( + "crypto/sha256" + "encoding/base64" + "errors" + "fmt" + "os" + "strings" + + "github.com/oracle/oci-go-sdk/v65/common" + "github.com/oracle/oci-go-sdk/v65/objectstorage" + "github.com/oracle/oci-go-sdk/v65/objectstorage/transfer" +) + +const ( + sseDefaultAlgorithm = "AES256" +) + +func getSha256(p []byte) []byte { + h := sha256.New() + h.Write(p) + return h.Sum(nil) +} + +func validateSSECustomerKeyOptions(opt *Options) error { + if opt.SSEKMSKeyID != "" && (opt.SSECustomerKeyFile != "" || opt.SSECustomerKey != "") { + return errors.New("oos: can't use vault sse_kms_key_id and local sse_customer_key at the same time") + } + if opt.SSECustomerKey != "" && opt.SSECustomerKeyFile != "" { + return errors.New("oos: can't use sse_customer_key and sse_customer_key_file at the same time") + } + if opt.SSEKMSKeyID != "" { + return nil + } + err := populateSSECustomerKeys(opt) + if err != nil { + return err + } + return nil +} + +func populateSSECustomerKeys(opt *Options) error { + if opt.SSECustomerKeyFile != "" { + // Reads the base64-encoded AES key data from the specified file and computes its SHA256 checksum + data, err := os.ReadFile(expandPath(opt.SSECustomerKeyFile)) + if err != nil { + return fmt.Errorf("oos: error reading sse_customer_key_file: %v", err) + } + opt.SSECustomerKey = strings.TrimSpace(string(data)) + } + if opt.SSECustomerKey != "" { + decoded, err := base64.StdEncoding.DecodeString(opt.SSECustomerKey) + if err != nil { + return fmt.Errorf("oos: Could not decode sse_customer_key_file: %w", err) + } + sha256Checksum := base64.StdEncoding.EncodeToString(getSha256(decoded)) + if opt.SSECustomerKeySha256 == "" { + opt.SSECustomerKeySha256 = sha256Checksum + } else { + if opt.SSECustomerKeySha256 != sha256Checksum { + return fmt.Errorf("the computed SHA256 checksum "+ + "(%v) of the key doesn't match the config entry sse_customer_key_sha256=(%v)", + sha256Checksum, opt.SSECustomerKeySha256) + } + } + if opt.SSECustomerAlgorithm == "" { + opt.SSECustomerAlgorithm = sseDefaultAlgorithm + } + } + return nil +} + +// https://docs.oracle.com/en-us/iaas/Content/Object/Tasks/usingyourencryptionkeys.htm +func useBYOKPutObject(fs *Fs, request *objectstorage.PutObjectRequest) { + if fs.opt.SSEKMSKeyID != "" { + request.OpcSseKmsKeyId = common.String(fs.opt.SSEKMSKeyID) + } + if fs.opt.SSECustomerAlgorithm != "" { + request.OpcSseCustomerAlgorithm = common.String(fs.opt.SSECustomerAlgorithm) + } + if fs.opt.SSECustomerKey != "" { + request.OpcSseCustomerKey = common.String(fs.opt.SSECustomerKey) + } + if fs.opt.SSECustomerKeySha256 != "" { + request.OpcSseCustomerKeySha256 = common.String(fs.opt.SSECustomerKeySha256) + } +} + +func useBYOKHeadObject(fs *Fs, request *objectstorage.HeadObjectRequest) { + if fs.opt.SSECustomerAlgorithm != "" { + request.OpcSseCustomerAlgorithm = common.String(fs.opt.SSECustomerAlgorithm) + } + if fs.opt.SSECustomerKey != "" { + request.OpcSseCustomerKey = common.String(fs.opt.SSECustomerKey) + } + if fs.opt.SSECustomerKeySha256 != "" { + request.OpcSseCustomerKeySha256 = common.String(fs.opt.SSECustomerKeySha256) + } +} + +func useBYOKGetObject(fs *Fs, request *objectstorage.GetObjectRequest) { + if fs.opt.SSECustomerAlgorithm != "" { + request.OpcSseCustomerAlgorithm = common.String(fs.opt.SSECustomerAlgorithm) + } + if fs.opt.SSECustomerKey != "" { + request.OpcSseCustomerKey = common.String(fs.opt.SSECustomerKey) + } + if fs.opt.SSECustomerKeySha256 != "" { + request.OpcSseCustomerKeySha256 = common.String(fs.opt.SSECustomerKeySha256) + } +} + +func useBYOKCopyObject(fs *Fs, request *objectstorage.CopyObjectRequest) { + if fs.opt.SSEKMSKeyID != "" { + request.OpcSseKmsKeyId = common.String(fs.opt.SSEKMSKeyID) + } + if fs.opt.SSECustomerAlgorithm != "" { + request.OpcSseCustomerAlgorithm = common.String(fs.opt.SSECustomerAlgorithm) + } + if fs.opt.SSECustomerKey != "" { + request.OpcSseCustomerKey = common.String(fs.opt.SSECustomerKey) + } + if fs.opt.SSECustomerKeySha256 != "" { + request.OpcSseCustomerKeySha256 = common.String(fs.opt.SSECustomerKeySha256) + } +} + +func useBYOKUpload(fs *Fs, request *transfer.UploadRequest) { + if fs.opt.SSEKMSKeyID != "" { + request.OpcSseKmsKeyId = common.String(fs.opt.SSEKMSKeyID) + } + if fs.opt.SSECustomerAlgorithm != "" { + request.OpcSseCustomerAlgorithm = common.String(fs.opt.SSECustomerAlgorithm) + } + if fs.opt.SSECustomerKey != "" { + request.OpcSseCustomerKey = common.String(fs.opt.SSECustomerKey) + } + if fs.opt.SSECustomerKeySha256 != "" { + request.OpcSseCustomerKeySha256 = common.String(fs.opt.SSECustomerKeySha256) + } +} diff --git a/backend/oracleobjectstorage/client.go b/backend/oracleobjectstorage/client.go index b5dee80f4..6be770d42 100644 --- a/backend/oracleobjectstorage/client.go +++ b/backend/oracleobjectstorage/client.go @@ -9,6 +9,8 @@ import ( "errors" "net/http" "os" + "path" + "strings" "github.com/oracle/oci-go-sdk/v65/common" "github.com/oracle/oci-go-sdk/v65/common/auth" @@ -18,15 +20,33 @@ import ( "github.com/rclone/rclone/fs/fshttp" ) +func expandPath(filepath string) (expandedPath string) { + if filepath == "" { + return filepath + } + cleanedPath := path.Clean(filepath) + expandedPath = cleanedPath + if strings.HasPrefix(cleanedPath, "~") { + rest := cleanedPath[2:] + home, err := os.UserHomeDir() + if err != nil { + return expandedPath + } + expandedPath = path.Join(home, rest) + } + return +} + func getConfigurationProvider(opt *Options) (common.ConfigurationProvider, error) { switch opt.Provider { case instancePrincipal: return auth.InstancePrincipalConfigurationProvider() case userPrincipal: - if opt.ConfigFile != "" && !fileExists(opt.ConfigFile) { - fs.Errorf(userPrincipal, "oci config file doesn't exist at %v", opt.ConfigFile) + expandConfigFilePath := expandPath(opt.ConfigFile) + if expandConfigFilePath != "" && !fileExists(expandConfigFilePath) { + fs.Errorf(userPrincipal, "oci config file doesn't exist at %v", expandConfigFilePath) } - return common.CustomProfileConfigProvider(opt.ConfigFile, opt.ConfigProfile), nil + return common.CustomProfileConfigProvider(expandConfigFilePath, opt.ConfigProfile), nil case resourcePrincipal: return auth.ResourcePrincipalConfigurationProvider() case noAuth: diff --git a/backend/oracleobjectstorage/copy.go b/backend/oracleobjectstorage/copy.go index 9ed0b768e..fec418fbd 100644 --- a/backend/oracleobjectstorage/copy.go +++ b/backend/oracleobjectstorage/copy.go @@ -74,6 +74,7 @@ func (f *Fs) copy(ctx context.Context, dstObj *Object, srcObj *Object) (err erro BucketName: common.String(srcBucket), CopyObjectDetails: copyObjectDetails, } + useBYOKCopyObject(f, &req) var resp objectstorage.CopyObjectResponse err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CopyObject(ctx, req) diff --git a/backend/oracleobjectstorage/object.go b/backend/oracleobjectstorage/object.go index d9f847923..e82ac295e 100644 --- a/backend/oracleobjectstorage/object.go +++ b/backend/oracleobjectstorage/object.go @@ -87,6 +87,7 @@ func (o *Object) headObject(ctx context.Context) (info *objectstorage.HeadObject BucketName: common.String(bucketName), ObjectName: common.String(objectPath), } + useBYOKHeadObject(o.fs, &req) var response objectstorage.HeadObjectResponse err = o.fs.pacer.Call(func() (bool, error) { var err error @@ -99,6 +100,7 @@ func (o *Object) headObject(ctx context.Context) (info *objectstorage.HeadObject return nil, fs.ErrorObjectNotFound } } + fs.Errorf(o, "Failed to head object: %v", err) return nil, err } o.fs.cache.MarkOK(bucketName) @@ -331,7 +333,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo ObjectName: common.String(bucketPath), } o.applyGetObjectOptions(&req, options...) - + useBYOKGetObject(o.fs, &req) var resp objectstorage.GetObjectResponse err := o.fs.pacer.Call(func() (bool, error) { var err error @@ -433,6 +435,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op uploadRequest.StorageTier = storageTier } o.applyMultiPutOptions(&uploadRequest, options...) + useBYOKUpload(o.fs, &uploadRequest) uploadStreamRequest := transfer.UploadStreamRequest{ UploadRequest: uploadRequest, StreamReader: in, @@ -506,8 +509,10 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op req.StorageTier = storageTier } o.applyPutOptions(&req, options...) + useBYOKPutObject(o.fs, &req) + var resp objectstorage.PutObjectResponse err = o.fs.pacer.Call(func() (bool, error) { - resp, err := o.fs.srv.PutObject(ctx, req) + resp, err = o.fs.srv.PutObject(ctx, req) return shouldRetry(ctx, resp.HTTPResponse(), err) }) if err != nil { diff --git a/backend/oracleobjectstorage/options.go b/backend/oracleobjectstorage/options.go index 004465093..9baf06e35 100644 --- a/backend/oracleobjectstorage/options.go +++ b/backend/oracleobjectstorage/options.go @@ -45,23 +45,28 @@ https://docs.oracle.com/en-us/iaas/Content/Identity/Tasks/callingservicesfromins // Options defines the configuration for this backend type Options struct { - Provider string `config:"provider"` - Compartment string `config:"compartment"` - Namespace string `config:"namespace"` - Region string `config:"region"` - Endpoint string `config:"endpoint"` - Enc encoder.MultiEncoder `config:"encoding"` - ConfigFile string `config:"config_file"` - ConfigProfile string `config:"config_profile"` - UploadCutoff fs.SizeSuffix `config:"upload_cutoff"` - ChunkSize fs.SizeSuffix `config:"chunk_size"` - UploadConcurrency int `config:"upload_concurrency"` - DisableChecksum bool `config:"disable_checksum"` - CopyCutoff fs.SizeSuffix `config:"copy_cutoff"` - CopyTimeout fs.Duration `config:"copy_timeout"` - StorageTier string `config:"storage_tier"` - LeavePartsOnError bool `config:"leave_parts_on_error"` - NoCheckBucket bool `config:"no_check_bucket"` + Provider string `config:"provider"` + Compartment string `config:"compartment"` + Namespace string `config:"namespace"` + Region string `config:"region"` + Endpoint string `config:"endpoint"` + Enc encoder.MultiEncoder `config:"encoding"` + ConfigFile string `config:"config_file"` + ConfigProfile string `config:"config_profile"` + UploadCutoff fs.SizeSuffix `config:"upload_cutoff"` + ChunkSize fs.SizeSuffix `config:"chunk_size"` + UploadConcurrency int `config:"upload_concurrency"` + DisableChecksum bool `config:"disable_checksum"` + CopyCutoff fs.SizeSuffix `config:"copy_cutoff"` + CopyTimeout fs.Duration `config:"copy_timeout"` + StorageTier string `config:"storage_tier"` + LeavePartsOnError bool `config:"leave_parts_on_error"` + NoCheckBucket bool `config:"no_check_bucket"` + SSEKMSKeyID string `config:"sse_kms_key_id"` + SSECustomerAlgorithm string `config:"sse_customer_algorithm"` + SSECustomerKey string `config:"sse_customer_key"` + SSECustomerKeyFile string `config:"sse_customer_key_file"` + SSECustomerKeySha256 string `config:"sse_customer_key_sha256"` } func newOptions() []fs.Option { @@ -252,5 +257,59 @@ creation permissions. `, Default: false, Advanced: true, + }, { + Name: "sse_customer_key_file", + Help: `To use SSE-C, a file containing the base64-encoded string of the AES-256 encryption key associated +with the object. Please note only one of sse_customer_key_file|sse_customer_key|sse_kms_key_id is needed.'`, + Advanced: true, + Examples: []fs.OptionExample{{ + Value: "", + Help: "None", + }}, + }, { + Name: "sse_customer_key", + Help: `To use SSE-C, the optional header that specifies the base64-encoded 256-bit encryption key to use to +encrypt or decrypt the data. Please note only one of sse_customer_key_file|sse_customer_key|sse_kms_key_id is +needed. For more information, see Using Your Own Keys for Server-Side Encryption +(https://docs.cloud.oracle.com/Content/Object/Tasks/usingyourencryptionkeys.htm)`, + Advanced: true, + Examples: []fs.OptionExample{{ + Value: "", + Help: "None", + }}, + }, { + Name: "sse_customer_key_sha256", + Help: `If using SSE-C, The optional header that specifies the base64-encoded SHA256 hash of the encryption +key. This value is used to check the integrity of the encryption key. see Using Your Own Keys for +Server-Side Encryption (https://docs.cloud.oracle.com/Content/Object/Tasks/usingyourencryptionkeys.htm).`, + Advanced: true, + Examples: []fs.OptionExample{{ + Value: "", + Help: "None", + }}, + }, { + Name: "sse_kms_key_id", + Help: `if using using your own master key in vault, this header specifies the +OCID (https://docs.cloud.oracle.com/Content/General/Concepts/identifiers.htm) of a master encryption key used to call +the Key Management service to generate a data encryption key or to encrypt or decrypt a data encryption key. +Please note only one of sse_customer_key_file|sse_customer_key|sse_kms_key_id is needed.`, + Advanced: true, + Examples: []fs.OptionExample{{ + Value: "", + Help: "None", + }}, + }, { + Name: "sse_customer_algorithm", + Help: `If using SSE-C, the optional header that specifies "AES256" as the encryption algorithm. +Object Storage supports "AES256" as the encryption algorithm. For more information, see +Using Your Own Keys for Server-Side Encryption (https://docs.cloud.oracle.com/Content/Object/Tasks/usingyourencryptionkeys.htm).`, + Advanced: true, + Examples: []fs.OptionExample{{ + Value: "", + Help: "None", + }, { + Value: sseDefaultAlgorithm, + Help: sseDefaultAlgorithm, + }}, }} } diff --git a/backend/oracleobjectstorage/oracleobjectstorage.go b/backend/oracleobjectstorage/oracleobjectstorage.go index c5f64744d..a0de006b8 100644 --- a/backend/oracleobjectstorage/oracleobjectstorage.go +++ b/backend/oracleobjectstorage/oracleobjectstorage.go @@ -59,6 +59,10 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e if err != nil { return nil, err } + err = validateSSECustomerKeyOptions(opt) + if err != nil { + return nil, err + } ci := fs.GetConfig(ctx) objectStorageClient, err := newObjectStorageClient(ctx, opt) if err != nil { diff --git a/docs/content/oracleobjectstorage.md b/docs/content/oracleobjectstorage.md index 75b3103b0..73cf1920d 100644 --- a/docs/content/oracleobjectstorage.md +++ b/docs/content/oracleobjectstorage.md @@ -561,6 +561,89 @@ Properties: - Type: bool - Default: false +#### --oos-sse-customer-key-file + +To use SSE-C, a file containing the base64-encoded string of the AES-256 encryption key associated +with the object. Please note only one of sse_customer_key_file|sse_customer_key|sse_kms_key_id is needed.' + +Properties: + +- Config: sse_customer_key_file +- Env Var: RCLONE_OOS_SSE_CUSTOMER_KEY_FILE +- Type: string +- Required: false +- Examples: + - "" + - None + +#### --oos-sse-customer-key + +To use SSE-C, the optional header that specifies the base64-encoded 256-bit encryption key to use to +encrypt or decrypt the data. Please note only one of sse_customer_key_file|sse_customer_key|sse_kms_key_id is +needed. For more information, see Using Your Own Keys for Server-Side Encryption +(https://docs.cloud.oracle.com/Content/Object/Tasks/usingyourencryptionkeys.htm) + +Properties: + +- Config: sse_customer_key +- Env Var: RCLONE_OOS_SSE_CUSTOMER_KEY +- Type: string +- Required: false +- Examples: + - "" + - None + +#### --oos-sse-customer-key-sha256 + +If using SSE-C, The optional header that specifies the base64-encoded SHA256 hash of the encryption +key. This value is used to check the integrity of the encryption key. see Using Your Own Keys for +Server-Side Encryption (https://docs.cloud.oracle.com/Content/Object/Tasks/usingyourencryptionkeys.htm). + +Properties: + +- Config: sse_customer_key_sha256 +- Env Var: RCLONE_OOS_SSE_CUSTOMER_KEY_SHA256 +- Type: string +- Required: false +- Examples: + - "" + - None + +#### --oos-sse-kms-key-id + +if using using your own master key in vault, this header specifies the +OCID (https://docs.cloud.oracle.com/Content/General/Concepts/identifiers.htm) of a master encryption key used to call +the Key Management service to generate a data encryption key or to encrypt or decrypt a data encryption key. +Please note only one of sse_customer_key_file|sse_customer_key|sse_kms_key_id is needed. + +Properties: + +- Config: sse_kms_key_id +- Env Var: RCLONE_OOS_SSE_KMS_KEY_ID +- Type: string +- Required: false +- Examples: + - "" + - None + +#### --oos-sse-customer-algorithm + +If using SSE-C, the optional header that specifies "AES256" as the encryption algorithm. +Object Storage supports "AES256" as the encryption algorithm. For more information, see +Using Your Own Keys for Server-Side Encryption (https://docs.cloud.oracle.com/Content/Object/Tasks/usingyourencryptionkeys.htm). + +Properties: + +- Config: sse_customer_algorithm +- Env Var: RCLONE_OOS_SSE_CUSTOMER_ALGORITHM +- Type: string +- Required: false +- Examples: + - "" + - None + - "AES256" + - AES256 + ## Backend commands Here are the commands specific to the oracleobjectstorage backend. diff --git a/go.mod b/go.mod index e6d46b80b..0bede5ecc 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( github.com/mitchellh/go-homedir v1.1.0 github.com/ncw/go-acd v0.0.0-20201019170801-fe55f33415b1 github.com/ncw/swift/v2 v2.0.1 - github.com/oracle/oci-go-sdk/v65 v65.28.0 + github.com/oracle/oci-go-sdk/v65 v65.30.0 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/sftp v1.13.6-0.20230213180117-971c283182b6 github.com/pmezard/go-difflib v1.0.0 diff --git a/go.sum b/go.sum index c4e402d54..fc85d31f6 100644 --- a/go.sum +++ b/go.sum @@ -358,8 +358,8 @@ github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/gomega v1.24.1 h1:KORJXNNTzJXzu4ScJWssJfJMnJ+2QJqhoQSRwNlze9E= -github.com/oracle/oci-go-sdk/v65 v65.28.0 h1:FgBWmHzT1y9H9tRQYOiX22Z8ec5p6/M5sSYRGcbWAno= -github.com/oracle/oci-go-sdk/v65 v65.28.0/go.mod h1:oyMrMa1vOzzKTmPN+kqrTR9y9kPA2tU1igN3NUSNTIE= +github.com/oracle/oci-go-sdk/v65 v65.30.0 h1:cP1IXZpJ0dxfDjFBulQm5YjA2pUjE83dyDinIp86rv0= +github.com/oracle/oci-go-sdk/v65 v65.30.0/go.mod h1:oyMrMa1vOzzKTmPN+kqrTR9y9kPA2tU1igN3NUSNTIE= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 h1:XeOYlK9W1uCmhjJSsY78Mcuh7MVkNjTzmHx1yBzizSU=