Remove flattening and replace with {off, standard} name encryption
This commit is contained in:
Nick Craig-Wood
parent
5f375a182d
commit
43eadf278c
151
crypt/cipher.go
151
crypt/cipher.go
|
|
@ -6,6 +6,7 @@ import (
|
|||
gocipher "crypto/cipher"
|
||||
"crypto/rand"
|
||||
"encoding/base32"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"sync"
|
||||
|
|
@ -30,6 +31,7 @@ const (
|
|||
blockHeaderSize = secretbox.Overhead
|
||||
blockDataSize = 64 * 1024
|
||||
blockSize = blockHeaderSize + blockDataSize
|
||||
encryptedSuffix = ".bin" // when file name encryption is off we add this suffix to make sure the cloud provider doesn't process the file
|
||||
)
|
||||
|
||||
// Errors returned by cipher
|
||||
|
|
@ -43,10 +45,8 @@ var (
|
|||
ErrorEncryptedBadMagic = errors.New("not an encrypted file - bad magic string")
|
||||
ErrorEncryptedBadBlock = errors.New("failed to authenticate decrypted block - bad password?")
|
||||
ErrorBadBase32Encoding = errors.New("bad base32 filename encoding")
|
||||
ErrorBadSpreadNotSingleChar = errors.New("bad unspread - not single character")
|
||||
ErrorBadSpreadResultTooShort = errors.New("bad unspread - result too short")
|
||||
ErrorBadSpreadDidntMatch = errors.New("bad unspread - directory prefix didn't match")
|
||||
ErrorFileClosed = errors.New("file already closed")
|
||||
ErrorNotAnEncryptedFile = errors.New("not an encrypted file - no \"" + encryptedSuffix + "\" suffix")
|
||||
defaultSalt = []byte{0xA8, 0x0D, 0xF4, 0x3A, 0x8F, 0xBD, 0x03, 0x08, 0xA7, 0xCA, 0xB8, 0x3E, 0x58, 0x1F, 0x86, 0xB1}
|
||||
)
|
||||
|
||||
|
|
@ -57,10 +57,14 @@ var (
|
|||
|
||||
// Cipher is used to swap out the encryption implementations
|
||||
type Cipher interface {
|
||||
// EncryptName encrypts a file path
|
||||
EncryptName(string) string
|
||||
// DecryptName decrypts a file path, returns error if decrypt was invalid
|
||||
DecryptName(string) (string, error)
|
||||
// EncryptFileName encrypts a file path
|
||||
EncryptFileName(string) string
|
||||
// DecryptFileName decrypts a file path, returns error if decrypt was invalid
|
||||
DecryptFileName(string) (string, error)
|
||||
// EncryptDirName encrypts a directory path
|
||||
EncryptDirName(string) string
|
||||
// DecryptDirName decrypts a directory path, returns error if decrypt was invalid
|
||||
DecryptDirName(string) (string, error)
|
||||
// EncryptData
|
||||
EncryptData(io.Reader) (io.Reader, error)
|
||||
// DecryptData
|
||||
|
|
@ -71,20 +75,56 @@ type Cipher interface {
|
|||
DecryptedSize(int64) (int64, error)
|
||||
}
|
||||
|
||||
// NameEncryptionMode is the type of file name encryption in use
|
||||
type NameEncryptionMode int
|
||||
|
||||
// NameEncryptionMode levels
|
||||
const (
|
||||
NameEncryptionOff NameEncryptionMode = iota
|
||||
NameEncryptionStandard
|
||||
)
|
||||
|
||||
// NewNameEncryptionMode turns a string into a NameEncryptionMode
|
||||
func NewNameEncryptionMode(s string) (mode NameEncryptionMode, err error) {
|
||||
s = strings.ToLower(s)
|
||||
switch s {
|
||||
case "off":
|
||||
mode = NameEncryptionOff
|
||||
case "standard":
|
||||
mode = NameEncryptionStandard
|
||||
default:
|
||||
err = errors.Errorf("Unknown file name encryption mode %q", s)
|
||||
}
|
||||
return mode, err
|
||||
}
|
||||
|
||||
// String turns mode into a human readable string
|
||||
func (mode NameEncryptionMode) String() (out string) {
|
||||
switch mode {
|
||||
case NameEncryptionOff:
|
||||
out = "off"
|
||||
case NameEncryptionStandard:
|
||||
out = "standard"
|
||||
default:
|
||||
out = fmt.Sprintf("Unknown mode #%d", mode)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type cipher struct {
|
||||
dataKey [32]byte // Key for secretbox
|
||||
nameKey [32]byte // 16,24 or 32 bytes
|
||||
nameTweak [nameCipherBlockSize]byte // used to tweak the name crypto
|
||||
block gocipher.Block
|
||||
flatten int // set flattening level - 0 is off
|
||||
mode NameEncryptionMode
|
||||
buffers sync.Pool // encrypt/decrypt buffers
|
||||
cryptoRand io.Reader // read crypto random numbers from here
|
||||
}
|
||||
|
||||
// newCipher initialises the cipher. If salt is "" then it uses a built in salt val
|
||||
func newCipher(flatten int, password, salt string) (*cipher, error) {
|
||||
func newCipher(mode NameEncryptionMode, password, salt string) (*cipher, error) {
|
||||
c := &cipher{
|
||||
flatten: flatten,
|
||||
mode: mode,
|
||||
cryptoRand: rand.Reader,
|
||||
}
|
||||
c.buffers.New = func() interface{} {
|
||||
|
|
@ -231,50 +271,8 @@ func (c *cipher) decryptSegment(ciphertext string) (string, error) {
|
|||
return string(plaintext), err
|
||||
}
|
||||
|
||||
// spread a name over the given number of directory levels
|
||||
//
|
||||
// if in isn't long enough dirs will be reduces
|
||||
func spreadName(dirs int, in string) string {
|
||||
if dirs > len(in) {
|
||||
dirs = len(in)
|
||||
}
|
||||
prefix := ""
|
||||
for i := 0; i < dirs; i++ {
|
||||
prefix += string(in[i]) + "/"
|
||||
}
|
||||
return prefix + in
|
||||
}
|
||||
|
||||
// reverse spreadName, returning an error if not in spread format
|
||||
//
|
||||
// This decodes any level of spreading
|
||||
func unspreadName(in string) (string, error) {
|
||||
in = strings.ToLower(in)
|
||||
segments := strings.Split(in, "/")
|
||||
if len(segments) == 0 {
|
||||
return in, nil
|
||||
}
|
||||
out := segments[len(segments)-1]
|
||||
segments = segments[:len(segments)-1]
|
||||
for i, s := range segments {
|
||||
if len(s) != 1 {
|
||||
return "", ErrorBadSpreadNotSingleChar
|
||||
}
|
||||
if i >= len(out) {
|
||||
return "", ErrorBadSpreadResultTooShort
|
||||
}
|
||||
if s[0] != out[i] {
|
||||
return "", ErrorBadSpreadDidntMatch
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// EncryptName encrypts a file path
|
||||
func (c *cipher) EncryptName(in string) string {
|
||||
if c.flatten > 0 {
|
||||
return spreadName(c.flatten, c.encryptSegment(in))
|
||||
}
|
||||
// encryptFileName encrypts a file path
|
||||
func (c *cipher) encryptFileName(in string) string {
|
||||
segments := strings.Split(in, "/")
|
||||
for i := range segments {
|
||||
segments[i] = c.encryptSegment(segments[i])
|
||||
|
|
@ -282,15 +280,24 @@ func (c *cipher) EncryptName(in string) string {
|
|||
return strings.Join(segments, "/")
|
||||
}
|
||||
|
||||
// DecryptName decrypts a file path
|
||||
func (c *cipher) DecryptName(in string) (string, error) {
|
||||
if c.flatten > 0 {
|
||||
unspread, err := unspreadName(in)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return c.decryptSegment(unspread)
|
||||
// EncryptFileName encrypts a file path
|
||||
func (c *cipher) EncryptFileName(in string) string {
|
||||
if c.mode == NameEncryptionOff {
|
||||
return in + encryptedSuffix
|
||||
}
|
||||
return c.encryptFileName(in)
|
||||
}
|
||||
|
||||
// EncryptDirName encrypts a directory path
|
||||
func (c *cipher) EncryptDirName(in string) string {
|
||||
if c.mode == NameEncryptionOff {
|
||||
return in
|
||||
}
|
||||
return c.encryptFileName(in)
|
||||
}
|
||||
|
||||
// decryptFileName decrypts a file path
|
||||
func (c *cipher) decryptFileName(in string) (string, error) {
|
||||
segments := strings.Split(in, "/")
|
||||
for i := range segments {
|
||||
var err error
|
||||
|
|
@ -302,6 +309,26 @@ func (c *cipher) DecryptName(in string) (string, error) {
|
|||
return strings.Join(segments, "/"), nil
|
||||
}
|
||||
|
||||
// DecryptFileName decrypts a file path
|
||||
func (c *cipher) DecryptFileName(in string) (string, error) {
|
||||
if c.mode == NameEncryptionOff {
|
||||
remainingLength := len(in) - len(encryptedSuffix)
|
||||
if remainingLength > 0 && strings.HasSuffix(in, encryptedSuffix) {
|
||||
return in[:remainingLength], nil
|
||||
}
|
||||
return "", ErrorNotAnEncryptedFile
|
||||
}
|
||||
return c.decryptFileName(in)
|
||||
}
|
||||
|
||||
// DecryptDirName decrypts a directory path
|
||||
func (c *cipher) DecryptDirName(in string) (string, error) {
|
||||
if c.mode == NameEncryptionOff {
|
||||
return in, nil
|
||||
}
|
||||
return c.decryptFileName(in)
|
||||
}
|
||||
|
||||
// nonce is an NACL secretbox nonce
|
||||
type nonce [fileNonceSize]byte
|
||||
|
||||
|
|
|
|||
|
|
@ -15,6 +15,32 @@ import (
|
|||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestNewNameEncryptionMode(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
in string
|
||||
expected NameEncryptionMode
|
||||
expectedErr string
|
||||
}{
|
||||
{"off", NameEncryptionOff, ""},
|
||||
{"standard", NameEncryptionStandard, ""},
|
||||
{"potato", NameEncryptionMode(0), "Unknown file name encryption mode \"potato\""},
|
||||
} {
|
||||
actual, actualErr := NewNameEncryptionMode(test.in)
|
||||
assert.Equal(t, actual, test.expected)
|
||||
if test.expectedErr == "" {
|
||||
assert.NoError(t, actualErr)
|
||||
} else {
|
||||
assert.Error(t, actualErr, test.expectedErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewNameEncryptionModeString(t *testing.T) {
|
||||
assert.Equal(t, NameEncryptionOff.String(), "off")
|
||||
assert.Equal(t, NameEncryptionStandard.String(), "standard")
|
||||
assert.Equal(t, NameEncryptionMode(2).String(), "Unknown mode #2")
|
||||
}
|
||||
|
||||
func TestValidString(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
in string
|
||||
|
|
@ -129,7 +155,7 @@ func TestDecodeFileName(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestEncryptSegment(t *testing.T) {
|
||||
c, _ := newCipher(0, "", "")
|
||||
c, _ := newCipher(NameEncryptionStandard, "", "")
|
||||
for _, test := range []struct {
|
||||
in string
|
||||
expected string
|
||||
|
|
@ -166,7 +192,7 @@ func TestEncryptSegment(t *testing.T) {
|
|||
|
||||
func TestDecryptSegment(t *testing.T) {
|
||||
// We've tested the forwards above, now concentrate on the errors
|
||||
c, _ := newCipher(0, "", "")
|
||||
c, _ := newCipher(NameEncryptionStandard, "", "")
|
||||
for _, test := range []struct {
|
||||
in string
|
||||
expectedErr error
|
||||
|
|
@ -184,87 +210,78 @@ func TestDecryptSegment(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestSpreadName(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
n int
|
||||
in string
|
||||
expected string
|
||||
}{
|
||||
{3, "", ""},
|
||||
{0, "abcdefg", "abcdefg"},
|
||||
{1, "abcdefg", "a/abcdefg"},
|
||||
{2, "abcdefg", "a/b/abcdefg"},
|
||||
{3, "abcdefg", "a/b/c/abcdefg"},
|
||||
{4, "abcdefg", "a/b/c/d/abcdefg"},
|
||||
{4, "abcd", "a/b/c/d/abcd"},
|
||||
{4, "abc", "a/b/c/abc"},
|
||||
{4, "ab", "a/b/ab"},
|
||||
{4, "a", "a/a"},
|
||||
} {
|
||||
actual := spreadName(test.n, test.in)
|
||||
assert.Equal(t, test.expected, actual, fmt.Sprintf("Testing %d,%q", test.n, test.in))
|
||||
recovered, err := unspreadName(test.expected)
|
||||
assert.NoError(t, err, fmt.Sprintf("Testing reverse %q", test.expected))
|
||||
assert.Equal(t, test.in, recovered, fmt.Sprintf("Testing reverse %q", test.expected))
|
||||
}
|
||||
func TestEncryptFileName(t *testing.T) {
|
||||
// First standard mode
|
||||
c, _ := newCipher(NameEncryptionStandard, "", "")
|
||||
assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s", c.EncryptFileName("1"))
|
||||
assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", c.EncryptFileName("1/12"))
|
||||
assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", c.EncryptFileName("1/12/123"))
|
||||
// Now off mode
|
||||
c, _ = newCipher(NameEncryptionOff, "", "")
|
||||
assert.Equal(t, "1/12/123.bin", c.EncryptFileName("1/12/123"))
|
||||
}
|
||||
|
||||
func TestUnspreadName(t *testing.T) {
|
||||
// We've tested the forwards above, now concentrate on the errors
|
||||
func TestDecryptFileName(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
in string
|
||||
expectedErr error
|
||||
}{
|
||||
{"aa/bc", ErrorBadSpreadNotSingleChar},
|
||||
{"/", ErrorBadSpreadNotSingleChar},
|
||||
{"a/", ErrorBadSpreadResultTooShort},
|
||||
{"a/b/c/ab", ErrorBadSpreadResultTooShort},
|
||||
{"a/b/x/abc", ErrorBadSpreadDidntMatch},
|
||||
{"a/b/c/ABC", nil},
|
||||
} {
|
||||
actual, actualErr := unspreadName(test.in)
|
||||
assert.Equal(t, test.expectedErr, actualErr, fmt.Sprintf("in=%q got actual=%q, err = %v %T", test.in, actual, actualErr, actualErr))
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncryptName(t *testing.T) {
|
||||
// First no flatten
|
||||
c, _ := newCipher(0, "", "")
|
||||
assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s", c.EncryptName("1"))
|
||||
assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", c.EncryptName("1/12"))
|
||||
assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", c.EncryptName("1/12/123"))
|
||||
// Now with flatten
|
||||
c, _ = newCipher(3, "", "")
|
||||
assert.Equal(t, "k/g/t/kgtickdcigo7600huebjl3ubu4", c.EncryptName("1/12/123"))
|
||||
}
|
||||
|
||||
func TestDecryptName(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
flatten int
|
||||
mode NameEncryptionMode
|
||||
in string
|
||||
expected string
|
||||
expectedErr error
|
||||
}{
|
||||
{0, "p0e52nreeaj0a5ea7s64m4j72s", "1", nil},
|
||||
{0, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", "1/12", nil},
|
||||
{0, "p0e52nreeAJ0A5EA7S64M4J72S/L42G6771HNv3an9cgc8cr2n1ng", "1/12", nil},
|
||||
{0, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", "1/12/123", nil},
|
||||
{0, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1/qgm4avr35m5loi1th53ato71v0", "", ErrorNotAMultipleOfBlocksize},
|
||||
{3, "k/g/t/kgtickdcigo7600huebjl3ubu4", "1/12/123", nil},
|
||||
{1, "k/g/t/kgtickdcigo7600huebjl3ubu4", "1/12/123", nil},
|
||||
{1, "k/g/t/i/kgtickdcigo7600huebjl3ubu4", "1/12/123", nil},
|
||||
{1, "k/x/t/i/kgtickdcigo7600huebjl3ubu4", "", ErrorBadSpreadDidntMatch},
|
||||
{NameEncryptionStandard, "p0e52nreeaj0a5ea7s64m4j72s", "1", nil},
|
||||
{NameEncryptionStandard, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", "1/12", nil},
|
||||
{NameEncryptionStandard, "p0e52nreeAJ0A5EA7S64M4J72S/L42G6771HNv3an9cgc8cr2n1ng", "1/12", nil},
|
||||
{NameEncryptionStandard, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", "1/12/123", nil},
|
||||
{NameEncryptionStandard, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1/qgm4avr35m5loi1th53ato71v0", "", ErrorNotAMultipleOfBlocksize},
|
||||
{NameEncryptionOff, "1/12/123.bin", "1/12/123", nil},
|
||||
{NameEncryptionOff, "1/12/123.bix", "", ErrorNotAnEncryptedFile},
|
||||
{NameEncryptionOff, ".bin", "", ErrorNotAnEncryptedFile},
|
||||
} {
|
||||
c, _ := newCipher(test.flatten, "", "")
|
||||
actual, actualErr := c.DecryptName(test.in)
|
||||
what := fmt.Sprintf("Testing %q (flatten=%d)", test.in, test.flatten)
|
||||
c, _ := newCipher(test.mode, "", "")
|
||||
actual, actualErr := c.DecryptFileName(test.in)
|
||||
what := fmt.Sprintf("Testing %q (mode=%v)", test.in, test.mode)
|
||||
assert.Equal(t, test.expected, actual, what)
|
||||
assert.Equal(t, test.expectedErr, actualErr, what)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncryptDirName(t *testing.T) {
|
||||
// First standard mode
|
||||
c, _ := newCipher(NameEncryptionStandard, "", "")
|
||||
assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s", c.EncryptDirName("1"))
|
||||
assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", c.EncryptDirName("1/12"))
|
||||
assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", c.EncryptDirName("1/12/123"))
|
||||
// Now off mode
|
||||
c, _ = newCipher(NameEncryptionOff, "", "")
|
||||
assert.Equal(t, "1/12/123", c.EncryptDirName("1/12/123"))
|
||||
}
|
||||
|
||||
func TestDecryptDirName(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
mode NameEncryptionMode
|
||||
in string
|
||||
expected string
|
||||
expectedErr error
|
||||
}{
|
||||
{NameEncryptionStandard, "p0e52nreeaj0a5ea7s64m4j72s", "1", nil},
|
||||
{NameEncryptionStandard, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", "1/12", nil},
|
||||
{NameEncryptionStandard, "p0e52nreeAJ0A5EA7S64M4J72S/L42G6771HNv3an9cgc8cr2n1ng", "1/12", nil},
|
||||
{NameEncryptionStandard, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", "1/12/123", nil},
|
||||
{NameEncryptionStandard, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1/qgm4avr35m5loi1th53ato71v0", "", ErrorNotAMultipleOfBlocksize},
|
||||
{NameEncryptionOff, "1/12/123.bin", "1/12/123.bin", nil},
|
||||
{NameEncryptionOff, "1/12/123", "1/12/123", nil},
|
||||
{NameEncryptionOff, ".bin", ".bin", nil},
|
||||
} {
|
||||
c, _ := newCipher(test.mode, "", "")
|
||||
actual, actualErr := c.DecryptDirName(test.in)
|
||||
what := fmt.Sprintf("Testing %q (mode=%v)", test.in, test.mode)
|
||||
assert.Equal(t, test.expected, actual, what)
|
||||
assert.Equal(t, test.expectedErr, actualErr, what)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncryptedSize(t *testing.T) {
|
||||
c, _ := newCipher(0, "", "")
|
||||
c, _ := newCipher(NameEncryptionStandard, "", "")
|
||||
for _, test := range []struct {
|
||||
in int64
|
||||
expected int64
|
||||
|
|
@ -288,7 +305,7 @@ func TestEncryptedSize(t *testing.T) {
|
|||
|
||||
func TestDecryptedSize(t *testing.T) {
|
||||
// Test the errors since we tested the reverse above
|
||||
c, _ := newCipher(0, "", "")
|
||||
c, _ := newCipher(NameEncryptionStandard, "", "")
|
||||
for _, test := range []struct {
|
||||
in int64
|
||||
expectedErr error
|
||||
|
|
@ -521,7 +538,7 @@ func (z *zeroes) Read(p []byte) (n int, err error) {
|
|||
|
||||
// Test encrypt decrypt with different buffer sizes
|
||||
func testEncryptDecrypt(t *testing.T, bufSize int, copySize int64) {
|
||||
c, err := newCipher(0, "", "")
|
||||
c, err := newCipher(NameEncryptionStandard, "", "")
|
||||
assert.NoError(t, err)
|
||||
c.cryptoRand = &zeroes{} // zero out the nonce
|
||||
buf := make([]byte, bufSize)
|
||||
|
|
@ -591,7 +608,7 @@ func TestEncryptData(t *testing.T) {
|
|||
{[]byte{1}, file1},
|
||||
{[]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}, file16},
|
||||
} {
|
||||
c, err := newCipher(0, "", "")
|
||||
c, err := newCipher(NameEncryptionStandard, "", "")
|
||||
assert.NoError(t, err)
|
||||
c.cryptoRand = newRandomSource(1E8) // nodge the crypto rand generator
|
||||
|
||||
|
|
@ -614,7 +631,7 @@ func TestEncryptData(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestNewEncrypter(t *testing.T) {
|
||||
c, err := newCipher(0, "", "")
|
||||
c, err := newCipher(NameEncryptionStandard, "", "")
|
||||
assert.NoError(t, err)
|
||||
c.cryptoRand = newRandomSource(1E8) // nodge the crypto rand generator
|
||||
|
||||
|
|
@ -658,7 +675,7 @@ func (c *closeDetector) Close() error {
|
|||
}
|
||||
|
||||
func TestNewDecrypter(t *testing.T) {
|
||||
c, err := newCipher(0, "", "")
|
||||
c, err := newCipher(NameEncryptionStandard, "", "")
|
||||
assert.NoError(t, err)
|
||||
c.cryptoRand = newRandomSource(1E8) // nodge the crypto rand generator
|
||||
|
||||
|
|
@ -700,7 +717,7 @@ func TestNewDecrypter(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestDecrypterRead(t *testing.T) {
|
||||
c, err := newCipher(0, "", "")
|
||||
c, err := newCipher(NameEncryptionStandard, "", "")
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Test truncating the header
|
||||
|
|
@ -744,7 +761,7 @@ func TestDecrypterRead(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestDecrypterClose(t *testing.T) {
|
||||
c, err := newCipher(0, "", "")
|
||||
c, err := newCipher(NameEncryptionStandard, "", "")
|
||||
assert.NoError(t, err)
|
||||
|
||||
cd := newCloseDetector(bytes.NewBuffer(file16))
|
||||
|
|
@ -780,7 +797,7 @@ func TestDecrypterClose(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestPutGetBlock(t *testing.T) {
|
||||
c, err := newCipher(0, "", "")
|
||||
c, err := newCipher(NameEncryptionStandard, "", "")
|
||||
assert.NoError(t, err)
|
||||
|
||||
block := c.getBlock()
|
||||
|
|
@ -791,7 +808,7 @@ func TestPutGetBlock(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestKey(t *testing.T) {
|
||||
c, err := newCipher(0, "", "")
|
||||
c, err := newCipher(NameEncryptionStandard, "", "")
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Check zero keys OK
|
||||
|
|
|
|||
116
crypt/crypt.go
116
crypt/crypt.go
|
|
@ -5,7 +5,6 @@ import (
|
|||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/ncw/rclone/fs"
|
||||
|
|
@ -22,27 +21,15 @@ func init() {
|
|||
Name: "remote",
|
||||
Help: "Remote to encrypt/decrypt.",
|
||||
}, {
|
||||
Name: "flatten",
|
||||
Help: "Flatten the directory structure - more secure, less useful - see docs for tradeoffs.",
|
||||
Name: "filename_encryption",
|
||||
Help: "How to encrypt the filenames.",
|
||||
Examples: []fs.OptionExample{
|
||||
{
|
||||
Value: "0",
|
||||
Help: "Don't flatten files (default) - good for unlimited files, but doesn't hide directory structure.",
|
||||
Value: "off",
|
||||
Help: "Don't encrypt the file names. Adds a \".bin\" extension only.",
|
||||
}, {
|
||||
Value: "1",
|
||||
Help: "Spread files over 1 directory good for <10,000 files.",
|
||||
}, {
|
||||
Value: "2",
|
||||
Help: "Spread files over 32 directories good for <320,000 files.",
|
||||
}, {
|
||||
Value: "3",
|
||||
Help: "Spread files over 1024 directories good for <10,000,000 files.",
|
||||
}, {
|
||||
Value: "4",
|
||||
Help: "Spread files over 32,768 directories good for <320,000,000 files.",
|
||||
}, {
|
||||
Value: "5",
|
||||
Help: "Spread files over 1,048,576 levels good for <10,000,000,000 files.",
|
||||
Value: "standard",
|
||||
Help: "Encrypt the filenames see the docs for the details.",
|
||||
},
|
||||
},
|
||||
}, {
|
||||
|
|
@ -60,12 +47,15 @@ func init() {
|
|||
|
||||
// NewFs contstructs an Fs from the path, container:path
|
||||
func NewFs(name, rpath string) (fs.Fs, error) {
|
||||
flatten := fs.ConfigFile.MustInt(name, "flatten", 0)
|
||||
mode, err := NewNameEncryptionMode(fs.ConfigFile.MustValue(name, "filename_encryption", "standard"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
password := fs.ConfigFile.MustValue(name, "password", "")
|
||||
if password == "" {
|
||||
return nil, errors.New("password not set in config file")
|
||||
}
|
||||
password, err := fs.Reveal(password)
|
||||
password, err = fs.Reveal(password)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to decrypt password")
|
||||
}
|
||||
|
|
@ -76,20 +66,26 @@ func NewFs(name, rpath string) (fs.Fs, error) {
|
|||
return nil, errors.Wrap(err, "failed to decrypt password2")
|
||||
}
|
||||
}
|
||||
cipher, err := newCipher(flatten, password, salt)
|
||||
cipher, err := newCipher(mode, password, salt)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to make cipher")
|
||||
}
|
||||
remote := fs.ConfigFile.MustValue(name, "remote")
|
||||
remotePath := path.Join(remote, cipher.EncryptName(rpath))
|
||||
// Look for a file first
|
||||
remotePath := path.Join(remote, cipher.EncryptFileName(rpath))
|
||||
wrappedFs, err := fs.NewFs(remotePath)
|
||||
// if that didn't produce a file, look for a directory
|
||||
if err != fs.ErrorIsFile {
|
||||
remotePath = path.Join(remote, cipher.EncryptDirName(rpath))
|
||||
wrappedFs, err = fs.NewFs(remotePath)
|
||||
}
|
||||
if err != fs.ErrorIsFile && err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to make remote %q to wrap", remotePath)
|
||||
}
|
||||
f := &Fs{
|
||||
Fs: wrappedFs,
|
||||
cipher: cipher,
|
||||
flatten: flatten,
|
||||
Fs: wrappedFs,
|
||||
cipher: cipher,
|
||||
mode: mode,
|
||||
}
|
||||
return f, err
|
||||
}
|
||||
|
|
@ -97,8 +93,8 @@ func NewFs(name, rpath string) (fs.Fs, error) {
|
|||
// Fs represents a wrapped fs.Fs
|
||||
type Fs struct {
|
||||
fs.Fs
|
||||
cipher Cipher
|
||||
flatten int
|
||||
cipher Cipher
|
||||
mode NameEncryptionMode
|
||||
}
|
||||
|
||||
// String returns a description of the FS
|
||||
|
|
@ -108,12 +104,12 @@ func (f *Fs) String() string {
|
|||
|
||||
// List the Fs into a channel
|
||||
func (f *Fs) List(opts fs.ListOpts, dir string) {
|
||||
f.Fs.List(f.newListOpts(opts, dir), f.cipher.EncryptName(dir))
|
||||
f.Fs.List(f.newListOpts(opts, dir), f.cipher.EncryptDirName(dir))
|
||||
}
|
||||
|
||||
// NewObject finds the Object at remote.
|
||||
func (f *Fs) NewObject(remote string) (fs.Object, error) {
|
||||
o, err := f.Fs.NewObject(f.cipher.EncryptName(remote))
|
||||
o, err := f.Fs.NewObject(f.cipher.EncryptFileName(remote))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -174,7 +170,7 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
|
|||
if !ok {
|
||||
return nil, fs.ErrorCantCopy
|
||||
}
|
||||
oResult, err := do.Copy(o.Object, f.cipher.EncryptName(remote))
|
||||
oResult, err := do.Copy(o.Object, f.cipher.EncryptFileName(remote))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -199,7 +195,7 @@ func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
|
|||
if !ok {
|
||||
return nil, fs.ErrorCantCopy
|
||||
}
|
||||
oResult, err := do.Move(o.Object, f.cipher.EncryptName(remote))
|
||||
oResult, err := do.Move(o.Object, f.cipher.EncryptFileName(remote))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -242,7 +238,7 @@ func (o *Object) String() string {
|
|||
// Remote returns the remote path
|
||||
func (o *Object) Remote() string {
|
||||
remote := o.Object.Remote()
|
||||
decryptedName, err := o.f.cipher.DecryptName(remote)
|
||||
decryptedName, err := o.f.cipher.DecryptFileName(remote)
|
||||
if err != nil {
|
||||
fs.Debug(remote, "Undecryptable file name: %v", err)
|
||||
return remote
|
||||
|
|
@ -287,7 +283,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error {
|
|||
func (f *Fs) newDir(dir *fs.Dir) *fs.Dir {
|
||||
new := *dir
|
||||
remote := dir.Name
|
||||
decryptedRemote, err := f.cipher.DecryptName(remote)
|
||||
decryptedRemote, err := f.cipher.DecryptDirName(remote)
|
||||
if err != nil {
|
||||
fs.Debug(remote, "Undecryptable dir name: %v", err)
|
||||
} else {
|
||||
|
|
@ -318,7 +314,7 @@ func (o *ObjectInfo) Fs() fs.Info {
|
|||
|
||||
// Remote returns the remote path
|
||||
func (o *ObjectInfo) Remote() string {
|
||||
return o.f.cipher.EncryptName(o.ObjectInfo.Remote())
|
||||
return o.f.cipher.EncryptFileName(o.ObjectInfo.Remote())
|
||||
}
|
||||
|
||||
// Size returns the size of the file
|
||||
|
|
@ -358,55 +354,19 @@ func (f *Fs) newListOpts(lo fs.ListOpts, dir string) *ListOpts {
|
|||
//
|
||||
// Each returned item must have less than level `/`s in.
|
||||
func (lo *ListOpts) Level() int {
|
||||
// If flattened recurse fully
|
||||
if lo.f.flatten > 0 {
|
||||
return fs.MaxLevel
|
||||
}
|
||||
return lo.ListOpts.Level()
|
||||
}
|
||||
|
||||
// addSyntheticDirs makes up directory objects for the path passed in
|
||||
func (lo *ListOpts) addSyntheticDirs(path string) {
|
||||
lo.mu.Lock()
|
||||
defer lo.mu.Unlock()
|
||||
for {
|
||||
i := strings.LastIndexByte(path, '/')
|
||||
if i < 0 {
|
||||
break
|
||||
}
|
||||
path = path[:i]
|
||||
if path == "" {
|
||||
break
|
||||
}
|
||||
if _, found := lo.dirs[path]; found {
|
||||
break
|
||||
}
|
||||
slashes := strings.Count(path, "/")
|
||||
if slashes < lo.ListOpts.Level() {
|
||||
lo.ListOpts.AddDir(&fs.Dir{Name: path})
|
||||
}
|
||||
lo.dirs[path] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
// Add an object to the output.
|
||||
// If the function returns true, the operation has been aborted.
|
||||
// Multiple goroutines can safely add objects concurrently.
|
||||
func (lo *ListOpts) Add(obj fs.Object) (abort bool) {
|
||||
remote := obj.Remote()
|
||||
decryptedRemote, err := lo.f.cipher.DecryptName(remote)
|
||||
_, err := lo.f.cipher.DecryptFileName(remote)
|
||||
if err != nil {
|
||||
fs.Debug(remote, "Skipping undecryptable file name: %v", err)
|
||||
return lo.ListOpts.IsFinished()
|
||||
}
|
||||
// If flattened add synthetic directories
|
||||
if lo.f.flatten > 0 {
|
||||
lo.addSyntheticDirs(decryptedRemote)
|
||||
slashes := strings.Count(decryptedRemote, "/")
|
||||
if slashes >= lo.ListOpts.Level() {
|
||||
return lo.ListOpts.IsFinished()
|
||||
}
|
||||
}
|
||||
return lo.ListOpts.Add(lo.f.newObject(obj))
|
||||
}
|
||||
|
||||
|
|
@ -414,12 +374,8 @@ func (lo *ListOpts) Add(obj fs.Object) (abort bool) {
|
|||
// If the function returns true, the operation has been aborted.
|
||||
// Multiple goroutines can safely add objects concurrently.
|
||||
func (lo *ListOpts) AddDir(dir *fs.Dir) (abort bool) {
|
||||
// If flattened we don't add any directories from the underlying remote
|
||||
if lo.f.flatten > 0 {
|
||||
return lo.ListOpts.IsFinished()
|
||||
}
|
||||
remote := dir.Name
|
||||
_, err := lo.f.cipher.DecryptName(remote)
|
||||
_, err := lo.f.cipher.DecryptDirName(remote)
|
||||
if err != nil {
|
||||
fs.Debug(remote, "Skipping undecryptable dir name: %v", err)
|
||||
return lo.ListOpts.IsFinished()
|
||||
|
|
@ -430,11 +386,7 @@ func (lo *ListOpts) AddDir(dir *fs.Dir) (abort bool) {
|
|||
// IncludeDirectory returns whether this directory should be
|
||||
// included in the listing (and recursed into or not).
|
||||
func (lo *ListOpts) IncludeDirectory(remote string) bool {
|
||||
// If flattened we look in all directories
|
||||
if lo.f.flatten > 0 {
|
||||
return true
|
||||
}
|
||||
decryptedRemote, err := lo.f.cipher.DecryptName(remote)
|
||||
decryptedRemote, err := lo.f.cipher.DecryptDirName(remote)
|
||||
if err != nil {
|
||||
fs.Debug(remote, "Not including undecryptable directory name: %v", err)
|
||||
return false
|
||||
|
|
|
|||
|
|
@ -19,8 +19,8 @@ First check your chosen remote is working - we'll call it
|
|||
will be encrypted and anything outside won't. This means that if you
|
||||
are using a bucket based remote (eg S3, B2, swift) then you should
|
||||
probably put the bucket in the remote `s3:bucket`. If you just use
|
||||
`s3:` then rclone will make encrypted bucket names too which may or
|
||||
may not be what you want.
|
||||
`s3:` then rclone will make encrypted bucket names too (if using file
|
||||
name encryption) which may or may not be what you want.
|
||||
|
||||
Now configure `crypt` using `rclone config`. We will call this one
|
||||
`secret` to differentiate it from the `remote`.
|
||||
|
|
@ -30,7 +30,7 @@ No remotes found - make a new one
|
|||
n) New remote
|
||||
s) Set configuration password
|
||||
q) Quit config
|
||||
n/s/q> n
|
||||
n/s/q> n
|
||||
name> secret
|
||||
Type of storage to configure.
|
||||
Choose a number from below, or type in your own value
|
||||
|
|
@ -61,32 +61,44 @@ Choose a number from below, or type in your own value
|
|||
Storage> 5
|
||||
Remote to encrypt/decrypt.
|
||||
remote> remote:path
|
||||
Flatten the directory structure - more secure, less useful - see docs for tradeoffs.
|
||||
How to encrypt the filenames.
|
||||
Choose a number from below, or type in your own value
|
||||
1 / Don't flatten files (default) - good for unlimited files, but doesn't hide directory structure.
|
||||
\ "0"
|
||||
2 / Spread files over 1 directory good for <10,000 files.
|
||||
\ "1"
|
||||
3 / Spread files over 32 directories good for <320,000 files.
|
||||
\ "2"
|
||||
4 / Spread files over 1024 directories good for <10,000,000 files.
|
||||
\ "3"
|
||||
5 / Spread files over 32,768 directories good for <320,000,000 files.
|
||||
\ "4"
|
||||
6 / Spread files over 1,048,576 levels good for <10,000,000,000 files.
|
||||
\ "5"
|
||||
flatten> 1
|
||||
1 / Don't encrypt the file names. Adds a ".bin" extension only.
|
||||
\ "off"
|
||||
2 / Encrypt the filenames see the docs for the details.
|
||||
\ "standard"
|
||||
filename_encryption> 2
|
||||
Password or pass phrase for encryption.
|
||||
y) Yes type in my own password
|
||||
g) Generate random password
|
||||
y/g> y
|
||||
Enter the password:
|
||||
password:
|
||||
Confirm the password:
|
||||
password:
|
||||
Password or pass phrase for salt. Optional but recommended.
|
||||
Should be different to the previous password.
|
||||
y) Yes type in my own password
|
||||
g) Generate random password
|
||||
n) No leave this optional password blank
|
||||
y/g/n> g
|
||||
Password strength in bits.
|
||||
64 is just about memorable
|
||||
128 is secure
|
||||
1024 is the maximum
|
||||
Bits> 128
|
||||
Your password is: JAsJvRcgR-_veXNfy_sGmQ
|
||||
Use this password?
|
||||
y) Yes
|
||||
n) No
|
||||
y/n> y
|
||||
Remote config
|
||||
--------------------
|
||||
[secret]
|
||||
remote = remote:path
|
||||
flatten = 0
|
||||
password = 0_gtCJ422bzwAWP0UN2lggrjhA-sSg
|
||||
filename_encryption = standard
|
||||
password = CfDxopZIXFG0Oo-ac7dPLWWOHkNJbw
|
||||
password2 = HYUpfuzHJL8qnX9fOaIYijq0xnVLwyVzp3y4SF3TwYqAU6HLysk
|
||||
--------------------
|
||||
y) Yes this is OK
|
||||
e) Edit this remote
|
||||
|
|
@ -99,9 +111,9 @@ obscured so it isn't immediately obvious what it is. It is in no way
|
|||
secure unless you use config file encryption.
|
||||
|
||||
A long passphrase is recommended, or you can use a random one. Note
|
||||
that if you reconfigure rclone with the same password/passphrase
|
||||
that if you reconfigure rclone with the same passwords/passphrases
|
||||
elsewhere it will be compatible - all the secrets used are derived
|
||||
from that one password/passphrase.
|
||||
from those two passwords/passphrases.
|
||||
|
||||
Note that rclone does not encrypt
|
||||
* file length - this can be calcuated within 16 bytes
|
||||
|
|
@ -109,7 +121,8 @@ Note that rclone does not encrypt
|
|||
|
||||
## Example ##
|
||||
|
||||
To test I made a little directory of files
|
||||
To test I made a little directory of files using "standard" file name
|
||||
encryption.
|
||||
|
||||
```
|
||||
plaintext/
|
||||
|
|
@ -154,39 +167,43 @@ $ rclone -q ls secret:subdir
|
|||
10 subsubdir/file4.txt
|
||||
```
|
||||
|
||||
If you use the flattened flag then the listing will look and that last command will not work.
|
||||
If don't use file name encryption then the remote will look like this
|
||||
- note the `.bin` extensions added to prevent the cloud provider
|
||||
attempting to interpret the data.
|
||||
|
||||
```
|
||||
$ rclone -q ls remote:path
|
||||
56 t/tsdtcpdu6g9dpamn6poqc248tll9dj5ok78a363etmq8ushr821g
|
||||
57 g/gsrp2g0u85pgsi6kso74bjsrsafe11odpfln8qqpj6n9p20of0a0
|
||||
55 h/hagjclgavj2mbiqm6u6cnjjqcg
|
||||
58 4/4jsbao3dhi0jfoubt2oo493pbqmsshn92q01ddu7dg6428rlluhg
|
||||
54 v/v05749mltvv1tf4onltun46gls
|
||||
54 file0.txt.bin
|
||||
57 subdir/file3.txt.bin
|
||||
56 subdir/file2.txt.bin
|
||||
58 subdir/subsubdir/file4.txt.bin
|
||||
55 file1.txt.bin
|
||||
```
|
||||
|
||||
### Flattened vs non-Flattened ###
|
||||
### File name encryption modes ###
|
||||
|
||||
Pros and cons of each
|
||||
Here are some of the features of the file name encryption modes
|
||||
|
||||
Flattened
|
||||
* hides directory structures
|
||||
* identical file names won't have identical encrypted names
|
||||
* can't use a sub path
|
||||
* doesn't work: `rclone copy crypt:sub/dir /tmp/recovered`
|
||||
* use: `rclone copy --include "/sub/dir/**" crypt: /tmp/recovered`
|
||||
* will always have to recurse through the entire directory structure
|
||||
* can't copy a single file directly
|
||||
* doesn't work: `rclone copy crypt:path/to/file /tmp/recovered`
|
||||
* use: `rclone copy --include "/path/to/file" crypt: /tmp/recovered`
|
||||
Off
|
||||
* doesn't hide file names or directory structure
|
||||
* allows for longer file names (~246 characters)
|
||||
* can use sub paths and copy single files
|
||||
|
||||
Normal
|
||||
Standard
|
||||
* file names encrypted
|
||||
* file names can't be as long (~156 characters)
|
||||
* can use sub paths and copy single files
|
||||
* directory structure visibile
|
||||
* identical files names will have identical uploaded names
|
||||
* can use shortcuts to shorten the directory recursion
|
||||
|
||||
You can swap between flattened levels without re-uploading your files.
|
||||
Cloud storage systems have various limits on file name length and
|
||||
total path length which you are more likely to hit using "Standard"
|
||||
file name encryption. If you keep your file names to below 156
|
||||
characters in length then you should be OK on all providers.
|
||||
|
||||
There may be an even more secure file name encryption mode in the
|
||||
future which will address the long file name problem.
|
||||
|
||||
## File formats ##
|
||||
|
||||
|
|
@ -245,25 +262,23 @@ files.
|
|||
|
||||
### Name encryption ###
|
||||
|
||||
File names are encrypted by crypt. These are either encrypted segment
|
||||
by segment - the path is broken up into `/` separated strings and
|
||||
these are encrypted individually, or if working in flattened mode the
|
||||
whole path is encrypted `/`s and all.
|
||||
File names are encrypted segment by segment - the path is broken up
|
||||
into `/` separated strings and these are encrypted individually.
|
||||
|
||||
First file names are padded using using PKCS#7 to a multiple of 16
|
||||
bytes before encryption.
|
||||
File segments are padded using using PKCS#7 to a multiple of 16 bytes
|
||||
before encryption.
|
||||
|
||||
They are then encrypted with EME using AES with 256 bit key. EME
|
||||
(ECB-Mix-ECB) is a wide-block encryption mode presented in the 2003
|
||||
paper "A Parallelizable Enciphering Mode" by Halevi and Rogaway.
|
||||
|
||||
This makes for determinstic encryption which is what we want - the
|
||||
same filename must encrypt to the same thing.
|
||||
same filename must encrypt to the same thing otherwise we can't find
|
||||
it on the cloud storage system.
|
||||
|
||||
This means that
|
||||
|
||||
* filenames with the same name will encrypt the same
|
||||
* (though we can use directory flattening to avoid this if required)
|
||||
* filenames which start the same won't have a common prefix
|
||||
|
||||
This uses a 32 byte key (256 bits) and a 16 byte (128 bits) IV both of
|
||||
|
|
@ -281,8 +296,11 @@ used on case insensitive remotes (eg Windows, Amazon Drive).
|
|||
|
||||
### Key derivation ###
|
||||
|
||||
Rclone uses `scrypt` with parameters `N=16384, r=8, p=1` with a fixed
|
||||
salt to derive the 32+32+16 = 80 bytes of key material required.
|
||||
Rclone uses `scrypt` with parameters `N=16384, r=8, p=1` with a an
|
||||
optional user supplied salt (password2) to derive the 32+32+16 = 80
|
||||
bytes of key material required. If the user doesn't supply a salt
|
||||
then rclone uses an internal one.
|
||||
|
||||
`scrypt` makes it impractical to mount a dictionary attack on rclone
|
||||
encrypted data.
|
||||
encrypted data. For full protection agains this you should always use
|
||||
a salt.
|
||||
|
|
|
|||
Loading…
Reference in New Issue