From a4f1f3d4e805324ef2eaf89eb5d871bc36ae2704 Mon Sep 17 00:00:00 2001 From: Caleb Case Date: Thu, 4 Jun 2020 09:39:42 -0400 Subject: [PATCH] backend/tardigrade: Upgrade uplink to v1.0.7 This fixes a regression in the rclone tests from the v1.0.6 upgrade of uplink. The failure was due to an improperly converted error resulting in the wrong type of error. --- go.mod | 2 +- go.sum | 14 +- .../github.com/minio/sha256-simd/.travis.yml | 5 +- .../minio/sha256-simd/cpuid_other.go | 2 +- .../minio/sha256-simd/cpuid_others_arm64.go | 35 --- vendor/github.com/minio/sha256-simd/go.mod | 2 + .../sha256-simd/sha256blockAvx2_amd64.go | 2 +- .../sha256-simd/sha256blockAvx512_amd64.go | 10 +- .../minio/sha256-simd/sha256blockAvx_amd64.go | 2 +- .../minio/sha256-simd/sha256blockSha_amd64.go | 2 +- .../sha256-simd/sha256blockSsse_amd64.go | 2 +- .../minio/sha256-simd/sha256block_386.go | 25 -- .../minio/sha256-simd/sha256block_amd64.go | 2 +- .../minio/sha256-simd/sha256block_arm.go | 25 -- .../minio/sha256-simd/sha256block_arm64.go | 2 +- .../minio/sha256-simd/sha256block_arm64.s | 4 +- .../minio/sha256-simd/sha256block_other.go | 2 +- .../minio/sha256-simd/test-architectures.sh | 15 ++ vendor/modules.txt | 6 +- vendor/storj.io/common/pkcrypto/hashing.go | 4 +- .../storj.io/common/pkcrypto/hashing_std.go | 22 ++ vendor/storj.io/common/rpc/dial.go | 27 --- vendor/storj.io/uplink/LICENSE | 215 ++---------------- vendor/storj.io/uplink/README.md | 4 +- vendor/storj.io/uplink/bucket.go | 14 +- vendor/storj.io/uplink/config.go | 14 ++ vendor/storj.io/uplink/go.mod | 2 +- vendor/storj.io/uplink/go.sum | 11 +- vendor/storj.io/uplink/objects.go | 2 +- .../uplink/private/metainfo/client.go | 19 -- .../private/metainfo/kvmetainfo/buckets.go | 93 ++------ .../private/metainfo/kvmetainfo/metainfo.go | 4 +- .../private/metainfo/kvmetainfo/objects.go | 164 +------------ .../private/metainfo/kvmetainfo/temputils.go | 39 ---- .../uplink/private/storage/streams/size.go | 28 --- .../uplink/private/storage/streams/store.go | 2 +- .../uplink/private/stream/download.go | 21 +- vendor/storj.io/uplink/project.go | 6 +- 38 files changed, 138 insertions(+), 712 deletions(-) delete mode 100644 vendor/github.com/minio/sha256-simd/cpuid_others_arm64.go delete mode 100644 vendor/github.com/minio/sha256-simd/sha256block_386.go delete mode 100644 vendor/github.com/minio/sha256-simd/sha256block_arm.go create mode 100644 vendor/github.com/minio/sha256-simd/test-architectures.sh create mode 100644 vendor/storj.io/common/pkcrypto/hashing_std.go delete mode 100644 vendor/storj.io/uplink/private/metainfo/kvmetainfo/temputils.go delete mode 100644 vendor/storj.io/uplink/private/storage/streams/size.go diff --git a/go.mod b/go.mod index 070ead8e4..a4afb68ca 100644 --- a/go.mod +++ b/go.mod @@ -67,7 +67,7 @@ require ( google.golang.org/api v0.21.1-0.20200411000818-c8cf5cff125e google.golang.org/genproto v0.0.0-20200225123651-fc8f55426688 // indirect gopkg.in/yaml.v2 v2.2.8 - storj.io/uplink v1.0.6 + storj.io/uplink v1.0.7 ) go 1.14 diff --git a/go.sum b/go.sum index 528a50a54..6903361c2 100644 --- a/go.sum +++ b/go.sum @@ -260,8 +260,8 @@ github.com/mattn/go-runewidth v0.0.8 h1:3tS41NlGYSmhhe/8fhGRzc+z3AYCw1Fe1WAyLuuj github.com/mattn/go-runewidth v0.0.8/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/minio/sha256-simd v0.0.0-20190328051042-05b4dd3047e5 h1:l16XLUUJ34wIz+RIvLhSwGvLvKyy+W598b135bJN6mg= -github.com/minio/sha256-simd v0.0.0-20190328051042-05b4dd3047e5/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= +github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= +github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -356,7 +356,6 @@ github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:s github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spacemonkeygo/errors v0.0.0-20171212215202-9064522e9fd1 h1:xHQewZjohU9/wUsyC99navCjQDNHtTgUOM/J1jAbzfw= github.com/spacemonkeygo/errors v0.0.0-20171212215202-9064522e9fd1/go.mod h1:7NL9UAYQnRM5iKHUCld3tf02fKb5Dft+41+VckASUy0= -github.com/spacemonkeygo/monkit/v3 v3.0.0-20191108235033-eacca33b3037/go.mod h1:JcK1pCbReQsOsMKF/POFSZCq7drXFybgGmbc27tuwes= github.com/spacemonkeygo/monkit/v3 v3.0.4/go.mod h1:JcK1pCbReQsOsMKF/POFSZCq7drXFybgGmbc27tuwes= github.com/spacemonkeygo/monkit/v3 v3.0.5/go.mod h1:JcK1pCbReQsOsMKF/POFSZCq7drXFybgGmbc27tuwes= github.com/spacemonkeygo/monkit/v3 v3.0.7-0.20200515175308-072401d8c752 h1:WcQDknqg0qajLNYKv3mXgbkWlYs5rPgZehGJFWePHVI= @@ -404,7 +403,6 @@ github.com/youmark/pkcs8 v0.0.0-20191102193632-94c173a94d60 h1:Ud2neINE1YFEwrcJ4 github.com/youmark/pkcs8 v0.0.0-20191102193632-94c173a94d60/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yunify/qingstor-sdk-go/v3 v3.2.0 h1:9sB2WZMgjwSUNZhrgvaNGazVltoFUUfuS9f0uCWtTr8= github.com/yunify/qingstor-sdk-go/v3 v3.2.0/go.mod h1:KciFNuMu6F4WLk9nGwwK69sCGKLCdd9f97ac/wfumS4= -github.com/zeebo/admission/v2 v2.0.0/go.mod h1:gSeHGelDHW7Vq6UyJo2boeSt/6Dsnqpisv0i4YZSOyM= github.com/zeebo/admission/v3 v3.0.1/go.mod h1:BP3isIv9qa2A7ugEratNq1dnl2oZRXaQUGdU7WXKtbw= github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY= github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= @@ -657,9 +655,9 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -storj.io/common v0.0.0-20200519171747-3ff8acf78c46 h1:Yx73D928PKtyQYPXHuQ5WFES4t+0nufxbhwyf8VodMw= -storj.io/common v0.0.0-20200519171747-3ff8acf78c46/go.mod h1:hqUDJlDHU1kZuZmfLohWgGa0Cf3pL1IH8DsxLCsamNQ= +storj.io/common v0.0.0-20200529121635-ef4a5bc8ec88 h1:8Gy0vjF4Kj1n94n6mQrDU8T50W/rpwMI03udkATCUN0= +storj.io/common v0.0.0-20200529121635-ef4a5bc8ec88/go.mod h1:6S6Ub92/BB+ofU7hbyPcm96b4Q1ayyN0HLog+3u+wGc= storj.io/drpc v0.0.12 h1:4ei1M4cnWlYxcQheX0Dg4+c12zCD+oJqfweVQVWarsA= storj.io/drpc v0.0.12/go.mod h1:82nfl+6YwRwF6UG31cEWWUqv/FaKvP5SGqUvoqTxCMA= -storj.io/uplink v1.0.6 h1:UztG/bSJitJt2c5Ofq9SmQ+a/yrLiHcPVR9gLfzcWG0= -storj.io/uplink v1.0.6/go.mod h1:F+AeZR4l9F9A7K1K+GXJJarwaKOQK0RWmbyy/maMFEI= +storj.io/uplink v1.0.7 h1:vI0csamiYsN2vTj17x4QUUQ8xyyFVQvNzxJjUsSEi7U= +storj.io/uplink v1.0.7/go.mod h1:PiLAqaFPUgsOs4u0xfVtm1jkWSQwdQmXv7KQUJUL/MU= diff --git a/vendor/github.com/minio/sha256-simd/.travis.yml b/vendor/github.com/minio/sha256-simd/.travis.yml index 744e64cfd..4f85db539 100644 --- a/vendor/github.com/minio/sha256-simd/.travis.yml +++ b/vendor/github.com/minio/sha256-simd/.travis.yml @@ -7,7 +7,7 @@ os: go: - tip -- 1.11.x +- 1.12.x env: - ARCH=x86_64 @@ -21,4 +21,5 @@ matrix: script: - diff -au <(gofmt -d .) <(printf "") - go test -race -v ./... -- go tool vet -asmdecl . +- go vet -asmdecl . +- ./test-architectures.sh diff --git a/vendor/github.com/minio/sha256-simd/cpuid_other.go b/vendor/github.com/minio/sha256-simd/cpuid_other.go index 04f26ce88..3e4415828 100644 --- a/vendor/github.com/minio/sha256-simd/cpuid_other.go +++ b/vendor/github.com/minio/sha256-simd/cpuid_other.go @@ -13,7 +13,7 @@ // limitations under the License. // -// +build ppc64 ppc64le mips mipsle mips64 mips64le s390x wasm +// +build !386,!amd64,!arm,!arm64 arm64,!linux package sha256 diff --git a/vendor/github.com/minio/sha256-simd/cpuid_others_arm64.go b/vendor/github.com/minio/sha256-simd/cpuid_others_arm64.go deleted file mode 100644 index 0fb4022f7..000000000 --- a/vendor/github.com/minio/sha256-simd/cpuid_others_arm64.go +++ /dev/null @@ -1,35 +0,0 @@ -// +build arm64,!linux - -// Minio Cloud Storage, (C) 2016 Minio, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -package sha256 - -func cpuid(op uint32) (eax, ebx, ecx, edx uint32) { - return 0, 0, 0, 0 -} - -func cpuidex(op, op2 uint32) (eax, ebx, ecx, edx uint32) { - return 0, 0, 0, 0 -} - -func xgetbv(index uint32) (eax, edx uint32) { - return 0, 0 -} - -// Check for sha2 instruction flag. -func haveArmSha() bool { - return false -} diff --git a/vendor/github.com/minio/sha256-simd/go.mod b/vendor/github.com/minio/sha256-simd/go.mod index b68fb0a05..4451e9eb2 100644 --- a/vendor/github.com/minio/sha256-simd/go.mod +++ b/vendor/github.com/minio/sha256-simd/go.mod @@ -1 +1,3 @@ module github.com/minio/sha256-simd + +go 1.12 diff --git a/vendor/github.com/minio/sha256-simd/sha256blockAvx2_amd64.go b/vendor/github.com/minio/sha256-simd/sha256blockAvx2_amd64.go index 43ee7a948..52fcaee6d 100644 --- a/vendor/github.com/minio/sha256-simd/sha256blockAvx2_amd64.go +++ b/vendor/github.com/minio/sha256-simd/sha256blockAvx2_amd64.go @@ -1,4 +1,4 @@ -//+build !noasm +//+build !noasm,!appengine /* * Minio Cloud Storage, (C) 2016 Minio, Inc. diff --git a/vendor/github.com/minio/sha256-simd/sha256blockAvx512_amd64.go b/vendor/github.com/minio/sha256-simd/sha256blockAvx512_amd64.go index e6bd455df..db8e48d31 100644 --- a/vendor/github.com/minio/sha256-simd/sha256blockAvx512_amd64.go +++ b/vendor/github.com/minio/sha256-simd/sha256blockAvx512_amd64.go @@ -1,4 +1,4 @@ -//+build !noasm +//+build !noasm,!appengine /* * Minio Cloud Storage, (C) 2017 Minio, Inc. @@ -104,15 +104,16 @@ func (d *Avx512Digest) Sum(in []byte) (result []byte) { } trail := make([]byte, 0, 128) + trail = append(trail, d.x[:d.nx]...) len := d.len // Padding. Add a 1 bit and 0 bits until 56 bytes mod 64. var tmp [64]byte tmp[0] = 0x80 if len%64 < 56 { - trail = append(d.x[:d.nx], tmp[0:56-len%64]...) + trail = append(trail, tmp[0:56-len%64]...) } else { - trail = append(d.x[:d.nx], tmp[0:64+56-len%64]...) + trail = append(trail, tmp[0:64+56-len%64]...) } d.nx = 0 @@ -375,7 +376,7 @@ func (a512srv *Avx512Server) reset(uid uint64) { } // Invoke assembly and send results back -func (a512srv *Avx512Server) blocks() (err error) { +func (a512srv *Avx512Server) blocks() { inputs := [16][]byte{} for i := range inputs { @@ -397,7 +398,6 @@ func (a512srv *Avx512Server) blocks() (err error) { delete(a512srv.digests, uid) // Delete entry from hashmap } } - return } func (a512srv *Avx512Server) Write(uid uint64, p []byte) (nn int, err error) { diff --git a/vendor/github.com/minio/sha256-simd/sha256blockAvx_amd64.go b/vendor/github.com/minio/sha256-simd/sha256blockAvx_amd64.go index eb8a0ff0c..c2f71181f 100644 --- a/vendor/github.com/minio/sha256-simd/sha256blockAvx_amd64.go +++ b/vendor/github.com/minio/sha256-simd/sha256blockAvx_amd64.go @@ -1,4 +1,4 @@ -//+build !noasm +//+build !noasm,!appengine /* * Minio Cloud Storage, (C) 2016 Minio, Inc. diff --git a/vendor/github.com/minio/sha256-simd/sha256blockSha_amd64.go b/vendor/github.com/minio/sha256-simd/sha256blockSha_amd64.go index 383189c8c..483689ef0 100644 --- a/vendor/github.com/minio/sha256-simd/sha256blockSha_amd64.go +++ b/vendor/github.com/minio/sha256-simd/sha256blockSha_amd64.go @@ -1,4 +1,4 @@ -//+build !noasm +//+build !noasm,!appengine package sha256 diff --git a/vendor/github.com/minio/sha256-simd/sha256blockSsse_amd64.go b/vendor/github.com/minio/sha256-simd/sha256blockSsse_amd64.go index 54abbb0f0..1ae2320bd 100644 --- a/vendor/github.com/minio/sha256-simd/sha256blockSsse_amd64.go +++ b/vendor/github.com/minio/sha256-simd/sha256blockSsse_amd64.go @@ -1,4 +1,4 @@ -//+build !noasm +//+build !noasm,!appengine /* * Minio Cloud Storage, (C) 2016 Minio, Inc. diff --git a/vendor/github.com/minio/sha256-simd/sha256block_386.go b/vendor/github.com/minio/sha256-simd/sha256block_386.go deleted file mode 100644 index a4153b918..000000000 --- a/vendor/github.com/minio/sha256-simd/sha256block_386.go +++ /dev/null @@ -1,25 +0,0 @@ -//+build !noasm - -/* - * Minio Cloud Storage, (C) 2016 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package sha256 - -func blockArmGo(dig *digest, p []byte) {} -func blockAvx2Go(dig *digest, p []byte) {} -func blockAvxGo(dig *digest, p []byte) {} -func blockSsseGo(dig *digest, p []byte) {} -func blockShaGo(dig *digest, p []byte) {} diff --git a/vendor/github.com/minio/sha256-simd/sha256block_amd64.go b/vendor/github.com/minio/sha256-simd/sha256block_amd64.go index 8d341fcfc..1c4d97f0c 100644 --- a/vendor/github.com/minio/sha256-simd/sha256block_amd64.go +++ b/vendor/github.com/minio/sha256-simd/sha256block_amd64.go @@ -1,4 +1,4 @@ -//+build !noasm +//+build !noasm,!appengine /* * Minio Cloud Storage, (C) 2016 Minio, Inc. diff --git a/vendor/github.com/minio/sha256-simd/sha256block_arm.go b/vendor/github.com/minio/sha256-simd/sha256block_arm.go deleted file mode 100644 index 1191c0863..000000000 --- a/vendor/github.com/minio/sha256-simd/sha256block_arm.go +++ /dev/null @@ -1,25 +0,0 @@ -//+build !noasm - -/* - * Minio Cloud Storage, (C) 2016 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package sha256 - -func blockAvx2Go(dig *digest, p []byte) {} -func blockAvxGo(dig *digest, p []byte) {} -func blockSsseGo(dig *digest, p []byte) {} -func blockShaGo(dig *digest, p []byte) {} -func blockArmGo(dig *digest, p []byte) {} diff --git a/vendor/github.com/minio/sha256-simd/sha256block_arm64.go b/vendor/github.com/minio/sha256-simd/sha256block_arm64.go index 4441b0c23..0979c20ae 100644 --- a/vendor/github.com/minio/sha256-simd/sha256block_arm64.go +++ b/vendor/github.com/minio/sha256-simd/sha256block_arm64.go @@ -1,4 +1,4 @@ -//+build !noasm +//+build !noasm,!appengine /* * Minio Cloud Storage, (C) 2016 Minio, Inc. diff --git a/vendor/github.com/minio/sha256-simd/sha256block_arm64.s b/vendor/github.com/minio/sha256-simd/sha256block_arm64.s index db816ac6b..c6ddb3717 100644 --- a/vendor/github.com/minio/sha256-simd/sha256block_arm64.s +++ b/vendor/github.com/minio/sha256-simd/sha256block_arm64.s @@ -1,4 +1,4 @@ -//+build !noasm !appengine +//+build !noasm,!appengine // ARM64 version of SHA256 @@ -28,7 +28,7 @@ TEXT ·blockArm(SB), 7, $0 MOVD h+0(FP), R0 MOVD message+24(FP), R1 - MOVD lenmessage+32(FP), R2 // length of message + MOVD message_len+32(FP), R2 // length of message SUBS $64, R2 BMI complete diff --git a/vendor/github.com/minio/sha256-simd/sha256block_other.go b/vendor/github.com/minio/sha256-simd/sha256block_other.go index 0930d2a2c..0187c950a 100644 --- a/vendor/github.com/minio/sha256-simd/sha256block_other.go +++ b/vendor/github.com/minio/sha256-simd/sha256block_other.go @@ -1,4 +1,4 @@ -//+build noasm appengine ppc64 ppc64le mips mipsle mips64 mips64le s390x wasm +//+build appengine noasm !amd64,!arm64 /* * Minio Cloud Storage, (C) 2019 Minio, Inc. diff --git a/vendor/github.com/minio/sha256-simd/test-architectures.sh b/vendor/github.com/minio/sha256-simd/test-architectures.sh new file mode 100644 index 000000000..50150eaab --- /dev/null +++ b/vendor/github.com/minio/sha256-simd/test-architectures.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +set -e + +go tool dist list | while IFS=/ read os arch; do + echo "Checking $os/$arch..." + echo " normal" + GOARCH=$arch GOOS=$os go build -o /dev/null ./... + echo " noasm" + GOARCH=$arch GOOS=$os go build -tags noasm -o /dev/null ./... + echo " appengine" + GOARCH=$arch GOOS=$os go build -tags appengine -o /dev/null ./... + echo " noasm,appengine" + GOARCH=$arch GOOS=$os go build -tags 'appengine noasm' -o /dev/null ./... +done diff --git a/vendor/modules.txt b/vendor/modules.txt index 049a36843..5c26e8782 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -177,7 +177,7 @@ github.com/mattn/go-isatty github.com/mattn/go-runewidth # github.com/matttproud/golang_protobuf_extensions v1.0.1 github.com/matttproud/golang_protobuf_extensions/pbutil -# github.com/minio/sha256-simd v0.0.0-20190328051042-05b4dd3047e5 +# github.com/minio/sha256-simd v0.1.1 github.com/minio/sha256-simd # github.com/mitchellh/go-homedir v1.1.0 ## explicit @@ -465,7 +465,7 @@ google.golang.org/grpc/tap # gopkg.in/yaml.v2 v2.2.8 ## explicit gopkg.in/yaml.v2 -# storj.io/common v0.0.0-20200519171747-3ff8acf78c46 +# storj.io/common v0.0.0-20200529121635-ef4a5bc8ec88 storj.io/common/encryption storj.io/common/errs2 storj.io/common/fpath @@ -504,7 +504,7 @@ storj.io/drpc/drpcmux storj.io/drpc/drpcsignal storj.io/drpc/drpcstream storj.io/drpc/drpcwire -# storj.io/uplink v1.0.6 +# storj.io/uplink v1.0.7 ## explicit storj.io/uplink storj.io/uplink/internal/expose diff --git a/vendor/storj.io/common/pkcrypto/hashing.go b/vendor/storj.io/common/pkcrypto/hashing.go index 71ccd6299..06590989f 100644 --- a/vendor/storj.io/common/pkcrypto/hashing.go +++ b/vendor/storj.io/common/pkcrypto/hashing.go @@ -1,12 +1,14 @@ // Copyright (C) 2019 Storj Labs, Inc. // See LICENSE for copying information. +// +build !stdsha256 + package pkcrypto import ( "hash" - sha256 "github.com/minio/sha256-simd" + "github.com/minio/sha256-simd" ) // NewHash returns default hash in storj. diff --git a/vendor/storj.io/common/pkcrypto/hashing_std.go b/vendor/storj.io/common/pkcrypto/hashing_std.go new file mode 100644 index 000000000..da4b4eca4 --- /dev/null +++ b/vendor/storj.io/common/pkcrypto/hashing_std.go @@ -0,0 +1,22 @@ +// Copyright (C) 2020 Storj Labs, Inc. +// See LICENSE for copying information. + +// +build stdsha256 + +package pkcrypto + +import ( + "crypto/sha256" + "hash" +) + +// NewHash returns default hash in storj. +func NewHash() hash.Hash { + return sha256.New() +} + +// SHA256Hash calculates the SHA256 hash of the input data. +func SHA256Hash(data []byte) []byte { + sum := sha256.Sum256(data) + return sum[:] +} diff --git a/vendor/storj.io/common/rpc/dial.go b/vendor/storj.io/common/rpc/dial.go index 27f53b731..0ce1b59b5 100644 --- a/vendor/storj.io/common/rpc/dial.go +++ b/vendor/storj.io/common/rpc/dial.go @@ -16,7 +16,6 @@ import ( "storj.io/common/memory" "storj.io/common/netutil" - "storj.io/common/pb" "storj.io/common/peertls/tlsopts" "storj.io/common/rpc/rpcpool" "storj.io/common/rpc/rpctracing" @@ -140,32 +139,6 @@ func (d Dialer) dialContext(ctx context.Context, address string) (net.Conn, erro }, nil } -// DialNode creates an rpc connection to the specified node. -func (d Dialer) DialNode(ctx context.Context, node *pb.Node) (_ *Conn, err error) { - if node == nil { - return nil, Error.New("node is nil") - } - - defer mon.Task()(&ctx, "node: "+node.Id.String()[0:8])(&err) - - if d.TLSOptions == nil { - return nil, Error.New("tls options not set when required for this dial") - } - - return d.dial(ctx, node.GetAddress().GetAddress(), d.TLSOptions.ClientTLSConfig(node.Id)) -} - -// DialAddressID dials to the specified address and asserts it has the given node id. -func (d Dialer) DialAddressID(ctx context.Context, address string, id storj.NodeID) (_ *Conn, err error) { - defer mon.Task()(&ctx, "node: "+id.String()[0:8])(&err) - - if d.TLSOptions == nil { - return nil, Error.New("tls options not set when required for this dial") - } - - return d.dial(ctx, address, d.TLSOptions.ClientTLSConfig(id)) -} - // DialNodeURL dials to the specified node url and asserts it has the given node id. func (d Dialer) DialNodeURL(ctx context.Context, nodeURL storj.NodeURL) (_ *Conn, err error) { defer mon.Task()(&ctx, "node: "+nodeURL.ID.String()[0:8])(&err) diff --git a/vendor/storj.io/uplink/LICENSE b/vendor/storj.io/uplink/LICENSE index d64569567..8d869750c 100644 --- a/vendor/storj.io/uplink/LICENSE +++ b/vendor/storj.io/uplink/LICENSE @@ -1,202 +1,19 @@ +Copyright (c) 2020 Storj Labs, Inc. - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/storj.io/uplink/README.md b/vendor/storj.io/uplink/README.md index 7a632ca6d..5e440d1c8 100644 --- a/vendor/storj.io/uplink/README.md +++ b/vendor/storj.io/uplink/README.md @@ -24,7 +24,7 @@ retrieve those files! Our versioning in this repo is intended to primarily support the expectations of the [Go modules](https://blog.golang.org/using-go-modules) system, so you can expect that within a major version release, backwards-incompatible changes will be avoided at high -cost. This is not the case with our downstream repo, +cost. This is not the case with our downstream repo, [storj/storj](https://github.com/storj/storj/). # Documentation @@ -39,7 +39,7 @@ cost. This is not the case with our downstream repo, # License This library is distributed under the -[Apache v2](https://www.apache.org/licenses/LICENSE-2.0) license. +[MIT license](https://opensource.org/licenses/MIT) (also known as the Expat license). # Support diff --git a/vendor/storj.io/uplink/bucket.go b/vendor/storj.io/uplink/bucket.go index 02c9fc07e..6e04d6055 100644 --- a/vendor/storj.io/uplink/bucket.go +++ b/vendor/storj.io/uplink/bucket.go @@ -11,7 +11,6 @@ import ( "github.com/zeebo/errs" "storj.io/common/errs2" - "storj.io/common/memory" "storj.io/common/rpc/rpcstatus" "storj.io/common/storj" ) @@ -55,18 +54,7 @@ func (project *Project) StatBucket(ctx context.Context, bucket string) (info *Bu func (project *Project) CreateBucket(ctx context.Context, bucket string) (created *Bucket, err error) { defer mon.Func().RestartTrace(&ctx)(&err) - // TODO remove bucket configuration when proper fix will be deployed on satellite - b, err := project.project.CreateBucket(ctx, bucket, &storj.Bucket{ - PathCipher: storj.EncAESGCM, - DefaultRedundancyScheme: storj.RedundancyScheme{ - Algorithm: storj.ReedSolomon, - ShareSize: 256 * memory.B.Int32(), - RequiredShares: 29, - RepairShares: 35, - OptimalShares: 80, - TotalShares: 110, - }, - }) + b, err := project.project.CreateBucket(ctx, bucket) if err != nil { if storj.ErrNoBucket.Has(err) { diff --git a/vendor/storj.io/uplink/config.go b/vendor/storj.io/uplink/config.go index 848307746..2f5ba4b90 100644 --- a/vendor/storj.io/uplink/config.go +++ b/vendor/storj.io/uplink/config.go @@ -5,6 +5,7 @@ package uplink import ( "context" + "net" "time" "storj.io/common/identity" @@ -22,6 +23,16 @@ type Config struct { // DialTimeout defines how long client should wait for establishing // a connection to peers. DialTimeout time.Duration + + // DialContext is how sockets are opened and is called to establish + // a connection. If unset, net.Dialer is used. + DialContext func(ctx context.Context, network, address string) (net.Conn, error) +} + +type dialContextFunc func(ctx context.Context, network, address string) (net.Conn, error) + +func (f dialContextFunc) DialContext(ctx context.Context, network, address string) (net.Conn, error) { + return f(ctx, network, address) } func (config Config) dial(ctx context.Context, satelliteAddress string, apiKey *macaroon.APIKey) (_ *metainfo.Client, _ rpc.Dialer, fullNodeURL string, err error) { @@ -45,6 +56,9 @@ func (config Config) dial(ctx context.Context, satelliteAddress string, apiKey * dialer := rpc.NewDefaultDialer(tlsOptions) dialer.DialTimeout = config.DialTimeout + if config.DialContext != nil { + dialer.Transport = dialContextFunc(config.DialContext) + } nodeURL, err := storj.ParseNodeURL(satelliteAddress) if err != nil { diff --git a/vendor/storj.io/uplink/go.mod b/vendor/storj.io/uplink/go.mod index 3b620d973..d41a55947 100644 --- a/vendor/storj.io/uplink/go.mod +++ b/vendor/storj.io/uplink/go.mod @@ -11,5 +11,5 @@ require ( github.com/zeebo/errs v1.2.2 go.uber.org/zap v1.10.0 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e - storj.io/common v0.0.0-20200519171747-3ff8acf78c46 + storj.io/common v0.0.0-20200529121635-ef4a5bc8ec88 ) diff --git a/vendor/storj.io/uplink/go.sum b/vendor/storj.io/uplink/go.sum index ef208d087..f056da016 100644 --- a/vendor/storj.io/uplink/go.sum +++ b/vendor/storj.io/uplink/go.sum @@ -25,8 +25,8 @@ github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlT github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= -github.com/minio/sha256-simd v0.0.0-20190328051042-05b4dd3047e5 h1:l16XLUUJ34wIz+RIvLhSwGvLvKyy+W598b135bJN6mg= -github.com/minio/sha256-simd v0.0.0-20190328051042-05b4dd3047e5/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= +github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= +github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= @@ -36,7 +36,6 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/spacemonkeygo/errors v0.0.0-20171212215202-9064522e9fd1 h1:xHQewZjohU9/wUsyC99navCjQDNHtTgUOM/J1jAbzfw= github.com/spacemonkeygo/errors v0.0.0-20171212215202-9064522e9fd1/go.mod h1:7NL9UAYQnRM5iKHUCld3tf02fKb5Dft+41+VckASUy0= -github.com/spacemonkeygo/monkit/v3 v3.0.0-20191108235033-eacca33b3037/go.mod h1:JcK1pCbReQsOsMKF/POFSZCq7drXFybgGmbc27tuwes= github.com/spacemonkeygo/monkit/v3 v3.0.4 h1:Ay+PZirv+qfd4sqcT+X/U3BnC7AcIaqp/IXh0oV36k8= github.com/spacemonkeygo/monkit/v3 v3.0.4/go.mod h1:JcK1pCbReQsOsMKF/POFSZCq7drXFybgGmbc27tuwes= github.com/spacemonkeygo/monkit/v3 v3.0.5 h1:vMW8Ne6WAUU/OMYaSv7KGW9h/sRNgeh6TyBapOSuMhM= @@ -53,8 +52,6 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/vivint/infectious v0.0.0-20190108171102-2455b059135b h1:dLkqBELopfQNhe8S9ucnSf+HhiUCgK/hPIjVG0f9GlY= github.com/vivint/infectious v0.0.0-20190108171102-2455b059135b/go.mod h1:5oyMAv4hrBEKqBwORFsiqIrCNCmL2qcZLQTdJLYeYIc= -github.com/zeebo/admission/v2 v2.0.0 h1:220NPZzKmyfklysKFO95L7E2Gt5NwlxTWGE14VP8heE= -github.com/zeebo/admission/v2 v2.0.0/go.mod h1:gSeHGelDHW7Vq6UyJo2boeSt/6Dsnqpisv0i4YZSOyM= github.com/zeebo/admission/v3 v3.0.1 h1:/IWg2jLhfjBOUhhdKcbweSzcY3QlbbE57sqvU72EpqA= github.com/zeebo/admission/v3 v3.0.1/go.mod h1:BP3isIv9qa2A7ugEratNq1dnl2oZRXaQUGdU7WXKtbw= github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY= @@ -98,7 +95,7 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -storj.io/common v0.0.0-20200519171747-3ff8acf78c46 h1:Yx73D928PKtyQYPXHuQ5WFES4t+0nufxbhwyf8VodMw= -storj.io/common v0.0.0-20200519171747-3ff8acf78c46/go.mod h1:hqUDJlDHU1kZuZmfLohWgGa0Cf3pL1IH8DsxLCsamNQ= +storj.io/common v0.0.0-20200529121635-ef4a5bc8ec88 h1:8Gy0vjF4Kj1n94n6mQrDU8T50W/rpwMI03udkATCUN0= +storj.io/common v0.0.0-20200529121635-ef4a5bc8ec88/go.mod h1:6S6Ub92/BB+ofU7hbyPcm96b4Q1ayyN0HLog+3u+wGc= storj.io/drpc v0.0.12 h1:4ei1M4cnWlYxcQheX0Dg4+c12zCD+oJqfweVQVWarsA= storj.io/drpc v0.0.12/go.mod h1:82nfl+6YwRwF6UG31cEWWUqv/FaKvP5SGqUvoqTxCMA= diff --git a/vendor/storj.io/uplink/objects.go b/vendor/storj.io/uplink/objects.go index c42785e0c..6f640e9c5 100644 --- a/vendor/storj.io/uplink/objects.go +++ b/vendor/storj.io/uplink/objects.go @@ -96,7 +96,7 @@ func (objects *ObjectIterator) Next() bool { } func (objects *ObjectIterator) loadNext() bool { - list, err := objects.project.db.ListObjectsExtended(objects.ctx, objects.bucket, objects.options) + list, err := objects.project.db.ListObjects(objects.ctx, objects.bucket, objects.options) if err != nil { objects.err = convertKnownErrors(err, objects.bucket.Name, "") return false diff --git a/vendor/storj.io/uplink/private/metainfo/client.go b/vendor/storj.io/uplink/private/metainfo/client.go index 57914e1cf..5a6e777c1 100644 --- a/vendor/storj.io/uplink/private/metainfo/client.go +++ b/vendor/storj.io/uplink/private/metainfo/client.go @@ -54,21 +54,6 @@ func New(client pb.DRPCMetainfoClient, apiKey *macaroon.APIKey, userAgent string } } -// Dial dials to metainfo endpoint with the specified api key. -func Dial(ctx context.Context, dialer rpc.Dialer, address string, apiKey *macaroon.APIKey, userAgent string) (*Client, error) { - conn, err := dialer.DialAddressInsecureBestEffort(ctx, address) - if err != nil { - return nil, Error.Wrap(err) - } - - return &Client{ - conn: conn, - client: pb.NewDRPCMetainfoClient(conn), - apiKeyRaw: apiKey.SerializeRaw(), - userAgent: userAgent, - }, nil -} - // DialNodeURL dials to metainfo endpoint with the specified api key. func DialNodeURL(ctx context.Context, dialer rpc.Dialer, nodeURL string, apiKey *macaroon.APIKey, userAgent string) (*Client, error) { url, err := storj.ParseNodeURL(nodeURL) @@ -1175,10 +1160,6 @@ func (client *Client) Batch(ctx context.Context, requests ...BatchItem) (resp [] Requests: batchItems, }) if err != nil { - if errs2.IsRPC(err, rpcstatus.NotFound) { - return []BatchResponse{}, storj.ErrObjectNotFound.Wrap(err) - } - return []BatchResponse{}, Error.Wrap(err) } diff --git a/vendor/storj.io/uplink/private/metainfo/kvmetainfo/buckets.go b/vendor/storj.io/uplink/private/metainfo/kvmetainfo/buckets.go index 5123898c8..ade6fc9e0 100644 --- a/vendor/storj.io/uplink/private/metainfo/kvmetainfo/buckets.go +++ b/vendor/storj.io/uplink/private/metainfo/kvmetainfo/buckets.go @@ -6,123 +6,58 @@ package kvmetainfo import ( "context" - "github.com/zeebo/errs" - - "storj.io/common/encryption" "storj.io/common/storj" "storj.io/uplink/private/metainfo" ) // CreateBucket creates a new bucket. -func (db *Project) CreateBucket(ctx context.Context, bucketName string, info *storj.Bucket) (_ storj.Bucket, err error) { +func (db *Project) CreateBucket(ctx context.Context, bucketName string) (newBucket storj.Bucket, err error) { defer mon.Task()(&ctx)(&err) if bucketName == "" { return storj.Bucket{}, storj.ErrNoBucket.New("") } - if info == nil { - info = &storj.Bucket{PathCipher: storj.EncAESGCM} - } else { - // ensure we don't modify the input argument - clone := *info - info = &clone - } - - if info.DefaultSegmentsSize == 0 { - info.DefaultSegmentsSize = db.segmentsSize - } - - if !info.DefaultRedundancyScheme.IsZero() { - if err := validateBlockSize(info.DefaultRedundancyScheme, info.DefaultEncryptionParameters.BlockSize); err != nil { - return storj.Bucket{}, storj.ErrBucket.Wrap(err) - } - } - - if info.PathCipher < storj.EncNull || info.PathCipher > storj.EncSecretBox { - return storj.Bucket{}, encryption.ErrInvalidConfig.New("encryption type %d is not supported", info.PathCipher) - } - - info.Name = bucketName - - // uuid MarshalJSON implementation always returns err == nil - partnerID, _ := info.PartnerID.MarshalJSON() - newBucket, err := db.metainfo.CreateBucket(ctx, metainfo.CreateBucketParams{ - Name: []byte(info.Name), - PathCipher: info.PathCipher, - PartnerID: partnerID, - DefaultSegmentsSize: info.DefaultSegmentsSize, - DefaultRedundancyScheme: info.DefaultRedundancyScheme, - DefaultEncryptionParameters: info.DefaultEncryptionParameters, + newBucket, err = db.metainfo.CreateBucket(ctx, metainfo.CreateBucketParams{ + Name: []byte(bucketName), }) - if err != nil { - return storj.Bucket{}, storj.ErrBucket.Wrap(err) - } - - return newBucket, nil -} - -// validateBlockSize confirms the encryption block size aligns with stripe size. -// Stripes contain encrypted data therefore we want the stripe boundaries to match -// with the encryption block size boundaries. We also want stripes to be small for -// audits, but encryption can be a bit larger. All told, block size should be an integer -// multiple of stripe size. -func validateBlockSize(redundancyScheme storj.RedundancyScheme, blockSize int32) error { - stripeSize := redundancyScheme.StripeSize() - - if stripeSize == 0 || blockSize%stripeSize != 0 { - return errs.New("encryption BlockSize (%d) must be a multiple of RS ShareSize (%d) * RS RequiredShares (%d)", - blockSize, redundancyScheme.ShareSize, redundancyScheme.RequiredShares, - ) - } - return nil + return newBucket, storj.ErrBucket.Wrap(err) } // DeleteBucket deletes bucket. -func (db *Project) DeleteBucket(ctx context.Context, bucketName string) (_ storj.Bucket, err error) { +func (db *Project) DeleteBucket(ctx context.Context, bucketName string) (bucket storj.Bucket, err error) { defer mon.Task()(&ctx)(&err) if bucketName == "" { return storj.Bucket{}, storj.ErrNoBucket.New("") } - bucket, err := db.metainfo.DeleteBucket(ctx, metainfo.DeleteBucketParams{ + + bucket, err = db.metainfo.DeleteBucket(ctx, metainfo.DeleteBucketParams{ Name: []byte(bucketName), }) - if err != nil { - return storj.Bucket{}, storj.ErrBucket.Wrap(err) - } - - return bucket, nil + return bucket, storj.ErrBucket.Wrap(err) } // GetBucket gets bucket information. -func (db *Project) GetBucket(ctx context.Context, bucketName string) (_ storj.Bucket, err error) { +func (db *Project) GetBucket(ctx context.Context, bucketName string) (bucket storj.Bucket, err error) { defer mon.Task()(&ctx)(&err) if bucketName == "" { return storj.Bucket{}, storj.ErrNoBucket.New("") } - bucket, err := db.metainfo.GetBucket(ctx, metainfo.GetBucketParams{ + bucket, err = db.metainfo.GetBucket(ctx, metainfo.GetBucketParams{ Name: []byte(bucketName), }) - if err != nil { - return storj.Bucket{}, storj.ErrBucket.Wrap(err) - } - - return bucket, nil + return bucket, storj.ErrBucket.Wrap(err) } // ListBuckets lists buckets. -func (db *Project) ListBuckets(ctx context.Context, listOpts storj.BucketListOptions) (_ storj.BucketList, err error) { +func (db *Project) ListBuckets(ctx context.Context, listOpts storj.BucketListOptions) (bucketList storj.BucketList, err error) { defer mon.Task()(&ctx)(&err) - bucketList, err := db.metainfo.ListBuckets(ctx, metainfo.ListBucketsParams{ + bucketList, err = db.metainfo.ListBuckets(ctx, metainfo.ListBucketsParams{ ListOpts: listOpts, }) - if err != nil { - return storj.BucketList{}, storj.ErrBucket.Wrap(err) - } - - return bucketList, nil + return bucketList, storj.ErrBucket.Wrap(err) } diff --git a/vendor/storj.io/uplink/private/metainfo/kvmetainfo/metainfo.go b/vendor/storj.io/uplink/private/metainfo/kvmetainfo/metainfo.go index b53e9fc33..5c623ea6a 100644 --- a/vendor/storj.io/uplink/private/metainfo/kvmetainfo/metainfo.go +++ b/vendor/storj.io/uplink/private/metainfo/kvmetainfo/metainfo.go @@ -46,8 +46,8 @@ func New(project *Project, metainfo *metainfo.Client, streams streams.Store, seg } // CreateBucket creates a new bucket with the specified information. -func (db *DB) CreateBucket(ctx context.Context, bucketName string, info *storj.Bucket) (bucketInfo storj.Bucket, err error) { - return db.project.CreateBucket(ctx, bucketName, info) +func (db *DB) CreateBucket(ctx context.Context, bucketName string) (bucketInfo storj.Bucket, err error) { + return db.project.CreateBucket(ctx, bucketName) } // DeleteBucket deletes bucket. diff --git a/vendor/storj.io/uplink/private/metainfo/kvmetainfo/objects.go b/vendor/storj.io/uplink/private/metainfo/kvmetainfo/objects.go index 219c6742f..cc6c5cef1 100644 --- a/vendor/storj.io/uplink/private/metainfo/kvmetainfo/objects.go +++ b/vendor/storj.io/uplink/private/metainfo/kvmetainfo/objects.go @@ -9,7 +9,6 @@ import ( "strings" "storj.io/common/encryption" - "storj.io/common/memory" "storj.io/common/paths" "storj.io/common/pb" "storj.io/common/storj" @@ -18,23 +17,6 @@ import ( "storj.io/uplink/private/storage/streams" ) -// DefaultRS default values for RedundancyScheme. -var DefaultRS = storj.RedundancyScheme{ - Algorithm: storj.ReedSolomon, - RequiredShares: 20, - RepairShares: 30, - OptimalShares: 40, - TotalShares: 50, - ShareSize: 1 * memory.KiB.Int32(), -} - -// DefaultES default values for EncryptionParameters. -// BlockSize should default to the size of a stripe. -var DefaultES = storj.EncryptionParameters{ - CipherSuite: storj.EncAESGCM, - BlockSize: DefaultRS.StripeSize(), -} - var contentTypeKey = "content-type" // GetObject returns information about an object. @@ -92,23 +74,6 @@ func (db *DB) CreateObject(ctx context.Context, bucket storj.Bucket, path storj. // TODO: autodetect content type from the path extension // if info.ContentType == "" {} - if info.EncryptionParameters.IsZero() { - info.EncryptionParameters = storj.EncryptionParameters{ - CipherSuite: DefaultES.CipherSuite, - BlockSize: DefaultES.BlockSize, - } - } - - if info.RedundancyScheme.IsZero() { - info.RedundancyScheme = DefaultRS - - // If the provided EncryptionParameters.BlockSize isn't a multiple of the - // DefaultRS stripeSize, then overwrite the EncryptionParameters with the DefaultES values - if err := validateBlockSize(DefaultRS, info.EncryptionParameters.BlockSize); err != nil { - info.EncryptionParameters.BlockSize = DefaultES.BlockSize - } - } - return &mutableObject{ db: db, info: info, @@ -167,135 +132,8 @@ func (db *DB) ListObjects(ctx context.Context, bucket storj.Bucket, options stor return storj.ObjectList{}, storj.ErrNoBucket.New("") } - var startAfter string - switch options.Direction { - // TODO for now we are supporting only storj.After - // case storj.Forward: - // // forward lists forwards from cursor, including cursor - // startAfter = keyBefore(options.Cursor) - case storj.After: - // after lists forwards from cursor, without cursor - startAfter = options.Cursor - default: - return storj.ObjectList{}, errClass.New("invalid direction %d", options.Direction) - } - - // TODO: we should let libuplink users be able to determine what metadata fields they request as well - // metaFlags := meta.All - // if db.pathCipher(bucket) == storj.EncNull || db.pathCipher(bucket) == storj.EncNullBase64URL { - // metaFlags = meta.None - // } - - // TODO use flags with listing - // if metaFlags&meta.Size != 0 { - // Calculating the stream's size require also the user-defined metadata, - // where stream store keeps info about the number of segments and their size. - // metaFlags |= meta.UserDefined - // } - - prefix := streams.ParsePath(storj.JoinPaths(bucket.Name, options.Prefix)) - prefixKey, err := encryption.DerivePathKey(prefix.Bucket(), streams.PathForKey(prefix.UnencryptedPath().Raw()), db.encStore) - if err != nil { - return storj.ObjectList{}, errClass.Wrap(err) - } - - encPrefix, err := encryption.EncryptPathWithStoreCipher(prefix.Bucket(), prefix.UnencryptedPath(), db.encStore) - if err != nil { - return storj.ObjectList{}, errClass.Wrap(err) - } - - // If the raw unencrypted path ends in a `/` we need to remove the final - // section of the encrypted path. For example, if we are listing the path - // `/bob/`, the encrypted path results in `enc("")/enc("bob")/enc("")`. This - // is an incorrect list prefix, what we really want is `enc("")/enc("bob")` - if strings.HasSuffix(prefix.UnencryptedPath().Raw(), "/") { - lastSlashIdx := strings.LastIndex(encPrefix.Raw(), "/") - encPrefix = paths.NewEncrypted(encPrefix.Raw()[:lastSlashIdx]) - } - - // We have to encrypt startAfter but only if it doesn't contain a bucket. - // It contains a bucket if and only if the prefix has no bucket. This is why it is a raw - // string instead of a typed string: it's either a bucket or an unencrypted path component - // and that isn't known at compile time. - needsEncryption := prefix.Bucket() != "" - var base *encryption.Base - if needsEncryption { - _, _, base = db.encStore.LookupEncrypted(prefix.Bucket(), encPrefix) - - startAfter, err = encryption.EncryptPathRaw(startAfter, db.pathCipher(base.PathCipher), prefixKey) - if err != nil { - return storj.ObjectList{}, errClass.Wrap(err) - } - } - - items, more, err := db.metainfo.ListObjects(ctx, metainfo.ListObjectsParams{ - Bucket: []byte(bucket.Name), - EncryptedPrefix: []byte(encPrefix.Raw()), - EncryptedCursor: []byte(startAfter), - Limit: int32(options.Limit), - Recursive: options.Recursive, - }) - if err != nil { - return storj.ObjectList{}, errClass.Wrap(err) - } - - list = storj.ObjectList{ - Bucket: bucket.Name, - Prefix: options.Prefix, - More: more, - Items: make([]storj.Object, len(items)), - } - - for i, item := range items { - var path streams.Path - var itemPath string - - if needsEncryption { - itemPath, err = encryption.DecryptPathRaw(string(item.EncryptedPath), db.pathCipher(base.PathCipher), prefixKey) - if err != nil { - return storj.ObjectList{}, errClass.Wrap(err) - } - - // TODO(jeff): this shouldn't be necessary if we handled trailing slashes - // appropriately. there's some issues with list. - fullPath := prefix.UnencryptedPath().Raw() - if len(fullPath) > 0 && fullPath[len(fullPath)-1] != '/' { - fullPath += "/" - } - fullPath += itemPath - - path = streams.CreatePath(prefix.Bucket(), paths.NewUnencrypted(fullPath)) - } else { - itemPath = string(item.EncryptedPath) - path = streams.CreatePath(string(item.EncryptedPath), paths.Unencrypted{}) - } - - stream, streamMeta, err := streams.TypedDecryptStreamInfo(ctx, item.EncryptedMetadata, path, db.encStore) - if err != nil { - return storj.ObjectList{}, errClass.Wrap(err) - } - - object, err := objectFromMeta(bucket, itemPath, item, stream, streamMeta) - if err != nil { - return storj.ObjectList{}, errClass.Wrap(err) - } - - list.Items[i] = object - } - - return list, nil -} - -// ListObjectsExtended lists objects in bucket based on the ListOptions. -func (db *DB) ListObjectsExtended(ctx context.Context, bucket storj.Bucket, options storj.ListOptions) (list storj.ObjectList, err error) { - defer mon.Task()(&ctx)(&err) - - if bucket.Name == "" { - return storj.ObjectList{}, storj.ErrNoBucket.New("") - } - if options.Prefix != "" && !strings.HasSuffix(options.Prefix, "/") { - return storj.ObjectList{}, Error.New("prefix should end with slash") + return storj.ObjectList{}, errClass.New("prefix should end with slash") } var startAfter string diff --git a/vendor/storj.io/uplink/private/metainfo/kvmetainfo/temputils.go b/vendor/storj.io/uplink/private/metainfo/kvmetainfo/temputils.go deleted file mode 100644 index 4231de044..000000000 --- a/vendor/storj.io/uplink/private/metainfo/kvmetainfo/temputils.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright (C) 2019 Storj Labs, Inc. -// See LICENSE for copying information. - -package kvmetainfo - -import ( - "github.com/zeebo/errs" - - "storj.io/common/encryption" - "storj.io/common/memory" - "storj.io/common/storj" - "storj.io/uplink/private/metainfo" - "storj.io/uplink/private/storage/segments" - "storj.io/uplink/private/storage/streams" -) - -var ( - // Error is the errs class of SetupProject. - Error = errs.Class("SetupProject error") -) - -// SetupProject creates a project with temporary values until we can figure out how to bypass encryption related setup. -func SetupProject(m *metainfo.Client) (*Project, error) { - maxBucketMetaSize := 10 * memory.MiB - segment := segments.NewSegmentStore(m, nil) - - // volatile warning: we're setting an encryption key of all zeros for bucket - // metadata, when really the bucket metadata should be stored in a different - // system altogether. - // TODO: https://storjlabs.atlassian.net/browse/V3-1967 - encStore := encryption.NewStore() - encStore.SetDefaultKey(new(storj.Key)) - strms, err := streams.NewStreamStore(m, segment, maxBucketMetaSize.Int64(), encStore, memory.KiB.Int(), storj.EncAESGCM, maxBucketMetaSize.Int(), maxBucketMetaSize.Int64()) - if err != nil { - return nil, Error.New("failed to create streams: %v", err) - } - - return NewProject(strms, memory.KiB.Int32(), 64*memory.MiB.Int64(), *m), nil -} diff --git a/vendor/storj.io/uplink/private/storage/streams/size.go b/vendor/storj.io/uplink/private/storage/streams/size.go deleted file mode 100644 index 3fe6e898e..000000000 --- a/vendor/storj.io/uplink/private/storage/streams/size.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (C) 2019 Storj Labs, Inc. -// See LICENSE for copying information. - -package streams - -import "io" - -// SizeReader holds reader and size read so far. -type SizeReader struct { - reader io.Reader - size int64 -} - -// NewSizeReader keeps track of how much bytes are read from the reader. -func NewSizeReader(r io.Reader) *SizeReader { - return &SizeReader{reader: r} -} - -func (r *SizeReader) Read(p []byte) (n int, err error) { - n, err = r.reader.Read(p) - r.size += int64(n) - return n, err -} - -// Size returns the number of bytes read so far. -func (r *SizeReader) Size() int64 { - return r.size -} diff --git a/vendor/storj.io/uplink/private/storage/streams/store.go b/vendor/storj.io/uplink/private/storage/streams/store.go index 69c7ddcd9..01fab7fa8 100644 --- a/vendor/storj.io/uplink/private/storage/streams/store.go +++ b/vendor/storj.io/uplink/private/storage/streams/store.go @@ -155,7 +155,7 @@ func (s *streamStore) Put(ctx context.Context, path Path, data io.Reader, metada return Meta{}, err } - sizeReader := NewSizeReader(eofReader) + sizeReader := segments.SizeReader(eofReader) segmentReader := io.LimitReader(sizeReader, s.segmentSize) peekReader := NewPeekThresholdReader(segmentReader) // If the data is larger than the inline threshold size, then it will be a remote segment diff --git a/vendor/storj.io/uplink/private/stream/download.go b/vendor/storj.io/uplink/private/stream/download.go index f1d230fca..a00f81694 100644 --- a/vendor/storj.io/uplink/private/stream/download.go +++ b/vendor/storj.io/uplink/private/stream/download.go @@ -29,12 +29,15 @@ func NewDownload(ctx context.Context, stream kvmetainfo.ReadOnlyStream, streams ctx: ctx, stream: stream, streams: streams, - limit: -1, + limit: stream.Info().Size, } } // NewDownloadRange creates new stream range download with range from offset to offset+limit. func NewDownloadRange(ctx context.Context, stream kvmetainfo.ReadOnlyStream, streams streams.Store, offset, limit int64) *Download { + if limit < 0 { + limit = stream.Info().Size - offset + } return &Download{ ctx: ctx, stream: stream, @@ -56,22 +59,20 @@ func (download *Download) Read(data []byte) (n int, err error) { } if download.reader == nil { - err = download.resetReader(download.offset) + err = download.resetReader() if err != nil { return 0, err } } - if download.limit == 0 { + if download.limit <= 0 { return 0, io.EOF } - if download.limit > 0 && download.limit < int64(len(data)) { + if download.limit < int64(len(data)) { data = data[:download.limit] } n, err = download.reader.Read(data) - if download.limit >= 0 { - download.limit -= int64(n) - } + download.limit -= int64(n) download.offset += int64(n) return n, err @@ -92,7 +93,7 @@ func (download *Download) Close() error { return download.reader.Close() } -func (download *Download) resetReader(offset int64) error { +func (download *Download) resetReader() error { if download.reader != nil { err := download.reader.Close() if err != nil { @@ -107,12 +108,10 @@ func (download *Download) resetReader(offset int64) error { return err } - download.reader, err = rr.Range(download.ctx, offset, obj.Size-offset) + download.reader, err = rr.Range(download.ctx, download.offset, download.limit) if err != nil { return err } - download.offset = offset - return nil } diff --git a/vendor/storj.io/uplink/project.go b/vendor/storj.io/uplink/project.go index 4b56f163e..7cffbe955 100644 --- a/vendor/storj.io/uplink/project.go +++ b/vendor/storj.io/uplink/project.go @@ -73,11 +73,6 @@ func (config Config) OpenProject(ctx context.Context, access *Access) (project * return nil, packageError.Wrap(err) } - proj, err := kvmetainfo.SetupProject(metainfo) - if err != nil { - return nil, packageError.Wrap(err) - } - // TODO: All these should be controlled by the satellite and not configured by the uplink. // For now we need to have these hard coded values that match the satellite configuration // to be able to create the underlying ecclient, segement store and stream store. @@ -122,6 +117,7 @@ func (config Config) OpenProject(ctx context.Context, access *Access) (project * return nil, packageError.Wrap(err) } + proj := kvmetainfo.NewProject(streamStore, encBlockSize, segmentsSize, *metainfo) db := kvmetainfo.New(proj, metainfo, streamStore, segmentStore, access.encAccess.Store()) var eg errgroup.Group