diff --git a/fs/encodings/encodings.go b/fs/encodings/encodings.go index cd23c0b15..b4d193528 100644 --- a/fs/encodings/encodings.go +++ b/fs/encodings/encodings.go @@ -9,24 +9,14 @@ import ( "github.com/rclone/rclone/lib/encoder" ) -// Base only encodes the zero byte and slash -const Base = encoder.MultiEncoder( - encoder.EncodeZero | - encoder.EncodeSlash | - encoder.EncodeDot) - -// Display is the internal encoding for logging and output -const Display = encoder.Standard - // LocalUnix is the encoding used by the local backend for non windows platforms -const LocalUnix = Base +const LocalUnix = encoder.Base // LocalMacOS is the encoding used by the local backend for macOS // // macOS can't store invalid UTF-8, it converts them into %XX encoding -const LocalMacOS = encoder.MultiEncoder( - uint(Base) | - encoder.EncodeInvalidUtf8) +const LocalMacOS = (encoder.Base | + encoder.EncodeInvalidUtf8) // LocalWindows is the encoding used by the local backend for windows platforms // @@ -48,31 +38,28 @@ const LocalMacOS = encoder.MultiEncoder( // Also encode invalid UTF-8 bytes as Go can't convert them to UTF-16. // // https://docs.microsoft.com/de-de/windows/desktop/FileIO/naming-a-file#naming-conventions -const LocalWindows = encoder.MultiEncoder( - uint(Base) | - encoder.EncodeWin | - encoder.EncodeBackSlash | - encoder.EncodeCtl | - encoder.EncodeRightSpace | - encoder.EncodeRightPeriod | - encoder.EncodeInvalidUtf8) +const LocalWindows = (encoder.Base | + encoder.EncodeWin | + encoder.EncodeBackSlash | + encoder.EncodeCtl | + encoder.EncodeRightSpace | + encoder.EncodeRightPeriod | + encoder.EncodeInvalidUtf8) // AmazonCloudDrive is the encoding used by the amazonclouddrive backend // // Encode invalid UTF-8 bytes as json doesn't handle them properly. -const AmazonCloudDrive = encoder.MultiEncoder( - uint(Base) | - encoder.EncodeInvalidUtf8) +const AmazonCloudDrive = (encoder.Base | + encoder.EncodeInvalidUtf8) // B2 is the encoding used by the b2 backend // // See: https://www.backblaze.com/b2/docs/files.html // Encode invalid UTF-8 bytes as json doesn't handle them properly. // FIXME: allow /, but not leading, trailing or double -const B2 = encoder.MultiEncoder( - uint(Display) | - encoder.EncodeBackSlash | - encoder.EncodeInvalidUtf8) +const B2 = (encoder.Display | + encoder.EncodeBackSlash | + encoder.EncodeInvalidUtf8) // Box is the encoding used by the box backend // @@ -83,18 +70,16 @@ const B2 = encoder.MultiEncoder( // // Testing revealed names with leading spaces work fine. // Also encode invalid UTF-8 bytes as json doesn't handle them properly. -const Box = encoder.MultiEncoder( - uint(Display) | - encoder.EncodeBackSlash | - encoder.EncodeRightSpace | - encoder.EncodeInvalidUtf8) +const Box = (encoder.Display | + encoder.EncodeBackSlash | + encoder.EncodeRightSpace | + encoder.EncodeInvalidUtf8) // Drive is the encoding used by the drive backend // // Encode invalid UTF-8 bytes as json doesn't handle them properly. // Don't encode / as it's a valid name character in drive. -const Drive = encoder.MultiEncoder( - encoder.EncodeInvalidUtf8) +const Drive = (encoder.EncodeInvalidUtf8) // Dropbox is the encoding used by the dropbox backend // @@ -102,52 +87,46 @@ const Drive = encoder.MultiEncoder( // as invalid characters. // Testing revealed names with trailing spaces and the DEL character don't work. // Also encode invalid UTF-8 bytes as json doesn't handle them properly. -const Dropbox = encoder.MultiEncoder( - uint(Base) | - encoder.EncodeBackSlash | - encoder.EncodeDel | - encoder.EncodeRightSpace | - encoder.EncodeInvalidUtf8) +const Dropbox = (encoder.Base | + encoder.EncodeBackSlash | + encoder.EncodeDel | + encoder.EncodeRightSpace | + encoder.EncodeInvalidUtf8) // GoogleCloudStorage is the encoding used by the googlecloudstorage backend -const GoogleCloudStorage = encoder.MultiEncoder( - uint(Base) | - encoder.EncodeCrLf | - encoder.EncodeInvalidUtf8) +const GoogleCloudStorage = (encoder.Base | + encoder.EncodeCrLf | + encoder.EncodeInvalidUtf8) // JottaCloud is the encoding used by the jottacloud backend // // Encode invalid UTF-8 bytes as xml doesn't handle them properly. // // Also: '*', '/', ':', '<', '>', '?', '\"', '\x00', '|' -const JottaCloud = encoder.MultiEncoder( - uint(Display) | - encoder.EncodeWin | // :?"*<>| - encoder.EncodeInvalidUtf8) +const JottaCloud = (encoder.Display | + encoder.EncodeWin | // :?"*<>| + encoder.EncodeInvalidUtf8) // Koofr is the encoding used by the koofr backend // // Encode invalid UTF-8 bytes as json doesn't handle them properly. -const Koofr = encoder.MultiEncoder( - uint(Display) | - encoder.EncodeBackSlash | - encoder.EncodeInvalidUtf8) +const Koofr = (encoder.Display | + encoder.EncodeBackSlash | + encoder.EncodeInvalidUtf8) // Mailru is the encoding used by the mailru backend // // Encode invalid UTF-8 bytes as json doesn't handle them properly. -const Mailru = encoder.MultiEncoder( - uint(Display) | - encoder.EncodeWin | // :?"*<>| - encoder.EncodeBackSlash | - encoder.EncodeInvalidUtf8) +const Mailru = (encoder.Display | + encoder.EncodeWin | // :?"*<>| + encoder.EncodeBackSlash | + encoder.EncodeInvalidUtf8) // Mega is the encoding used by the mega backend // // Encode invalid UTF-8 bytes as json doesn't handle them properly. -const Mega = encoder.MultiEncoder( - uint(Base) | - encoder.EncodeInvalidUtf8) +const Mega = (encoder.Base | + encoder.EncodeInvalidUtf8) // OneDrive is the encoding used by the onedrive backend // @@ -184,16 +163,15 @@ const Mega = encoder.MultiEncoder( // the same rules as the Windows naming conventions. // // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/concepts/addressing-driveitems?view=odsp-graph-online#path-encoding -const OneDrive = encoder.MultiEncoder( - uint(Display) | - encoder.EncodeBackSlash | - encoder.EncodeHashPercent | - encoder.EncodeLeftSpace | - encoder.EncodeLeftTilde | - encoder.EncodeRightPeriod | - encoder.EncodeRightSpace | - encoder.EncodeWin | - encoder.EncodeInvalidUtf8) +const OneDrive = (encoder.Display | + encoder.EncodeBackSlash | + encoder.EncodeHashPercent | + encoder.EncodeLeftSpace | + encoder.EncodeLeftTilde | + encoder.EncodeRightPeriod | + encoder.EncodeRightSpace | + encoder.EncodeWin | + encoder.EncodeInvalidUtf8) // OpenDrive is the encoding used by the opendrive backend // @@ -218,44 +196,40 @@ const OneDrive = encoder.MultiEncoder( // Also encode invalid UTF-8 bytes as json doesn't handle them properly. // // https://www.opendrive.com/wp-content/uploads/guides/OpenDrive_API_guide.pdf -const OpenDrive = encoder.MultiEncoder( - uint(Base) | - encoder.EncodeWin | - encoder.EncodeLeftCrLfHtVt | - encoder.EncodeRightCrLfHtVt | - encoder.EncodeBackSlash | - encoder.EncodeLeftSpace | - encoder.EncodeRightSpace | - encoder.EncodeInvalidUtf8) +const OpenDrive = (encoder.Base | + encoder.EncodeWin | + encoder.EncodeLeftCrLfHtVt | + encoder.EncodeRightCrLfHtVt | + encoder.EncodeBackSlash | + encoder.EncodeLeftSpace | + encoder.EncodeRightSpace | + encoder.EncodeInvalidUtf8) // PremiumizeMe is the encoding used by the premiumizeme backend // // Encode invalid UTF-8 bytes as json doesn't handle them properly. -const PremiumizeMe = encoder.MultiEncoder( - uint(Display) | - encoder.EncodeBackSlash | - encoder.EncodeDoubleQuote | - encoder.EncodeInvalidUtf8) +const PremiumizeMe = (encoder.Display | + encoder.EncodeBackSlash | + encoder.EncodeDoubleQuote | + encoder.EncodeInvalidUtf8) // Pcloud is the encoding used by the pcloud backend // // Encode invalid UTF-8 bytes as json doesn't handle them properly. // // TODO: Investigate Unicode simplification (\ gets converted to \ server-side) -const Pcloud = encoder.MultiEncoder( - uint(Display) | - encoder.EncodeBackSlash | - encoder.EncodeInvalidUtf8) +const Pcloud = (encoder.Display | + encoder.EncodeBackSlash | + encoder.EncodeInvalidUtf8) // Putio is the encoding used by the putio backend // // Note that \ is renamed to - // // Encode invalid UTF-8 bytes as json doesn't handle them properly. -const Putio = encoder.MultiEncoder( - uint(Display) | - encoder.EncodeBackSlash | - encoder.EncodeInvalidUtf8) +const Putio = (encoder.Display | + encoder.EncodeBackSlash | + encoder.EncodeInvalidUtf8) // Fichier is the encoding used by the fichier backend // @@ -270,17 +244,16 @@ const Putio = encoder.MultiEncoder( // '`': '`', // FULLWIDTH GRAVE ACCENT // // Leading space and trailing space -const Fichier = encoder.MultiEncoder( - uint(Display) | - encoder.EncodeBackSlash | - encoder.EncodeSingleQuote | - encoder.EncodeBackQuote | - encoder.EncodeDoubleQuote | - encoder.EncodeLtGt | - encoder.EncodeDollar | - encoder.EncodeLeftSpace | - encoder.EncodeRightSpace | - encoder.EncodeInvalidUtf8) +const Fichier = (encoder.Display | + encoder.EncodeBackSlash | + encoder.EncodeSingleQuote | + encoder.EncodeBackQuote | + encoder.EncodeDoubleQuote | + encoder.EncodeLtGt | + encoder.EncodeDollar | + encoder.EncodeLeftSpace | + encoder.EncodeRightSpace | + encoder.EncodeInvalidUtf8) // FTP is the encoding used by the ftp backend // @@ -289,9 +262,8 @@ const Fichier = encoder.MultiEncoder( // // proftpd can't handle '*' in file names // pureftpd can't handle '[', ']' or '*' -const FTP = encoder.MultiEncoder( - uint(Display) | - encoder.EncodeRightSpace) +const FTP = (encoder.Display | + encoder.EncodeRightSpace) // S3 is the encoding used by the s3 backend // @@ -305,58 +277,52 @@ const FTP = encoder.MultiEncoder( // - doubled / encoding // - trailing / encoding // so that AWS keys are always valid file names -const S3 = encoder.MultiEncoder( - encoder.EncodeInvalidUtf8 | - encoder.EncodeSlash | - encoder.EncodeDot) +const S3 = (encoder.EncodeInvalidUtf8 | + encoder.EncodeSlash | + encoder.EncodeDot) // Swift is the encoding used by the swift backend -const Swift = encoder.MultiEncoder( - encoder.EncodeInvalidUtf8 | - encoder.EncodeSlash) +const Swift = (encoder.EncodeInvalidUtf8 | + encoder.EncodeSlash) // AzureBlob is the encoding used by the azureblob backend -const AzureBlob = encoder.MultiEncoder( - encoder.EncodeInvalidUtf8 | - encoder.EncodeSlash | - encoder.EncodeCtl | - encoder.EncodeDel | - encoder.EncodeBackSlash | - encoder.EncodeRightPeriod) +const AzureBlob = (encoder.EncodeInvalidUtf8 | + encoder.EncodeSlash | + encoder.EncodeCtl | + encoder.EncodeDel | + encoder.EncodeBackSlash | + encoder.EncodeRightPeriod) // QingStor is the encoding used by the qingstor backend -const QingStor = encoder.MultiEncoder( - encoder.EncodeInvalidUtf8 | - encoder.EncodeCtl | - encoder.EncodeSlash) +const QingStor = (encoder.EncodeInvalidUtf8 | + encoder.EncodeCtl | + encoder.EncodeSlash) // Sharefile is the encoding used by the sharefile backend -const Sharefile = encoder.MultiEncoder( - uint(Base) | - encoder.EncodeWin | // :?"*<>| - encoder.EncodeBackSlash | // \ - encoder.EncodeCtl | - encoder.EncodeRightSpace | - encoder.EncodeRightPeriod | - encoder.EncodeLeftSpace | - encoder.EncodeLeftPeriod | - encoder.EncodeInvalidUtf8) +const Sharefile = (encoder.Base | + encoder.EncodeWin | // :?"*<>| + encoder.EncodeBackSlash | // \ + encoder.EncodeCtl | + encoder.EncodeRightSpace | + encoder.EncodeRightPeriod | + encoder.EncodeLeftSpace | + encoder.EncodeLeftPeriod | + encoder.EncodeInvalidUtf8) // Yandex is the encoding used by the yandex backend // // Of the control characters \t \n \r are allowed // it doesn't seem worth making an exception for this -const Yandex = encoder.MultiEncoder( - uint(Display) | - encoder.EncodeInvalidUtf8) +const Yandex = (encoder.Display | + encoder.EncodeInvalidUtf8) // ByName returns the encoder for a give backend name or nil func ByName(name string) encoder.Encoder { switch strings.ToLower(name) { case "base": - return Base + return encoder.Base case "display": - return Display + return encoder.Display case "amazonclouddrive": return AmazonCloudDrive case "azureblob": diff --git a/lib/encoder/encoder.go b/lib/encoder/encoder.go index ecb2f8c8d..9d2019a99 100644 --- a/lib/encoder/encoder.go +++ b/lib/encoder/encoder.go @@ -37,32 +37,32 @@ const ( // Possible flags for the MultiEncoder const ( - EncodeZero uint = 0 // NUL(0x00) - EncodeSlash uint = 1 << iota // / - EncodeLtGt // <> - EncodeDoubleQuote // " - EncodeSingleQuote // ' - EncodeBackQuote // ` - EncodeDollar // $ - EncodeColon // : - EncodeQuestion // ? - EncodeAsterisk // * - EncodePipe // | - EncodeHash // # - EncodePercent // % - EncodeBackSlash // \ - EncodeCrLf // CR(0x0D), LF(0x0A) - EncodeDel // DEL(0x7F) - EncodeCtl // CTRL(0x01-0x1F) - EncodeLeftSpace // Leading SPACE - EncodeLeftPeriod // Leading . - EncodeLeftTilde // Leading ~ - EncodeLeftCrLfHtVt // Leading CR LF HT VT - EncodeRightSpace // Trailing SPACE - EncodeRightPeriod // Trailing . - EncodeRightCrLfHtVt // Trailing CR LF HT VT - EncodeInvalidUtf8 // Invalid UTF-8 bytes - EncodeDot // . and .. names + EncodeZero MultiEncoder = 0 // NUL(0x00) + EncodeSlash MultiEncoder = 1 << iota // / + EncodeLtGt // <> + EncodeDoubleQuote // " + EncodeSingleQuote // ' + EncodeBackQuote // ` + EncodeDollar // $ + EncodeColon // : + EncodeQuestion // ? + EncodeAsterisk // * + EncodePipe // | + EncodeHash // # + EncodePercent // % + EncodeBackSlash // \ + EncodeCrLf // CR(0x0D), LF(0x0A) + EncodeDel // DEL(0x7F) + EncodeCtl // CTRL(0x01-0x1F) + EncodeLeftSpace // Leading SPACE + EncodeLeftPeriod // Leading . + EncodeLeftTilde // Leading ~ + EncodeLeftCrLfHtVt // Leading CR LF HT VT + EncodeRightSpace // Trailing SPACE + EncodeRightPeriod // Trailing . + EncodeRightCrLfHtVt // Trailing CR LF HT VT + EncodeInvalidUtf8 // Invalid UTF-8 bytes + EncodeDot // . and .. names // Synthetic EncodeWin = EncodeColon | EncodeQuestion | EncodeDoubleQuote | EncodeAsterisk | EncodeLtGt | EncodePipe // :?"*<>| @@ -70,8 +70,8 @@ const ( ) // Has returns true if flag is contained in mask -func (mask MultiEncoder) Has(flag uint) bool { - return uint(mask)&flag != 0 +func (mask MultiEncoder) Has(flag MultiEncoder) bool { + return mask&flag != 0 } // Encoder can transform names to and from the original and translated version. diff --git a/lib/encoder/encoder_test.go b/lib/encoder/encoder_test.go index b041aaca4..df70b6f33 100644 --- a/lib/encoder/encoder_test.go +++ b/lib/encoder/encoder_test.go @@ -19,7 +19,7 @@ var ( func TestEncodeString(t *testing.T) { for _, test := range []struct { - mask uint + mask MultiEncoder want string }{ {0, "None"}, @@ -31,7 +31,7 @@ func TestEncodeString(t *testing.T) { {EncodeSlash | EncodeDollar | EncodeColon, "Slash,Dollar,Colon"}, {EncodeSlash | (1 << 31), "Slash,0x80000000"}, } { - got := MultiEncoder(test.mask).String() + got := test.mask.String() assert.Equal(t, test.want, got) } @@ -40,7 +40,7 @@ func TestEncodeString(t *testing.T) { func TestEncodeSet(t *testing.T) { for _, test := range []struct { in string - want uint + want MultiEncoder wantErr bool }{ {"", 0, true}, @@ -58,20 +58,20 @@ func TestEncodeSet(t *testing.T) { var got MultiEncoder err := got.Set(test.in) assert.Equal(t, test.wantErr, err != nil, err) - assert.Equal(t, MultiEncoder(test.want), got, test.in) + assert.Equal(t, test.want, got, test.in) } } type testCase struct { - mask uint + mask MultiEncoder in string out string } func TestEncodeSingleMask(t *testing.T) { for i, tc := range testCasesSingle { - e := MultiEncoder(tc.mask) + e := tc.mask t.Run(strconv.FormatInt(int64(i), 10), func(t *testing.T) { got := e.Encode(tc.in) if got != tc.out { @@ -87,7 +87,7 @@ func TestEncodeSingleMask(t *testing.T) { func TestEncodeSingleMaskEdge(t *testing.T) { for i, tc := range testCasesSingleEdge { - e := MultiEncoder(tc.mask) + e := tc.mask t.Run(strconv.FormatInt(int64(i), 10), func(t *testing.T) { got := e.Encode(tc.in) if got != tc.out { @@ -103,7 +103,7 @@ func TestEncodeSingleMaskEdge(t *testing.T) { func TestEncodeDoubleMaskEdge(t *testing.T) { for i, tc := range testCasesDoubleEdge { - e := MultiEncoder(tc.mask) + e := tc.mask t.Run(strconv.FormatInt(int64(i), 10), func(t *testing.T) { got := e.Encode(tc.in) if got != tc.out { @@ -161,7 +161,7 @@ func TestEncodeInvalidUnicode(t *testing.T) { out: "a\xBF\\xFEb", }, } { - e := MultiEncoder(tc.mask) + e := tc.mask t.Run(strconv.FormatInt(int64(i), 10), func(t *testing.T) { got := e.Encode(tc.in) if got != tc.out { @@ -203,7 +203,7 @@ func TestEncodeDot(t *testing.T) { out: ". .", }, } { - e := MultiEncoder(tc.mask) + e := tc.mask t.Run(strconv.FormatInt(int64(i), 10), func(t *testing.T) { got := e.Encode(tc.in) if got != tc.out { @@ -245,7 +245,7 @@ func TestDecodeHalf(t *testing.T) { out: "a‛B\\‛Eg", }, } { - e := MultiEncoder(tc.mask) + e := tc.mask t.Run(strconv.FormatInt(int64(i), 10), func(t *testing.T) { got := e.Decode(tc.in) if got != tc.out { @@ -255,16 +255,15 @@ func TestDecodeHalf(t *testing.T) { } } -const oneDrive = MultiEncoder( - uint(Standard) | - EncodeWin | - EncodeBackSlash | - EncodeHashPercent | - EncodeDel | - EncodeCtl | - EncodeLeftTilde | - EncodeRightSpace | - EncodeRightPeriod) +const oneDrive = (Standard | + EncodeWin | + EncodeBackSlash | + EncodeHashPercent | + EncodeDel | + EncodeCtl | + EncodeLeftTilde | + EncodeRightSpace | + EncodeRightPeriod) var benchTests = []struct { in string diff --git a/lib/encoder/internal/gen/main.go b/lib/encoder/internal/gen/main.go index 4a396cfd2..30c9e7c21 100644 --- a/lib/encoder/internal/gen/main.go +++ b/lib/encoder/internal/gen/main.go @@ -20,7 +20,7 @@ const ( ) type mapping struct { - mask uint + mask encoder.MultiEncoder src, dst []rune } type stringPair struct { @@ -36,7 +36,7 @@ package encoder ` var maskBits = []struct { - mask uint + mask encoder.MultiEncoder name string }{ {encoder.EncodeZero, "EncodeZero"}, @@ -68,7 +68,7 @@ var maskBits = []struct { } type edge struct { - mask uint + mask encoder.MultiEncoder name string edge int orig []rune @@ -429,7 +429,7 @@ func fatalW(_ int, err error) func(...interface{}) { return func(s ...interface{}) {} } -func invalidMask(mask uint) bool { +func invalidMask(mask encoder.MultiEncoder) bool { return mask&(encoder.EncodeCtl|encoder.EncodeCrLf) != 0 && mask&(encoder.EncodeLeftCrLfHtVt|encoder.EncodeRightCrLfHtVt) != 0 } @@ -445,7 +445,7 @@ func runeRange(l, h rune) []rune { return out } -func getMapping(mask uint) mapping { +func getMapping(mask encoder.MultiEncoder) mapping { for _, m := range allMappings { if m.mask == mask { return m diff --git a/lib/encoder/standard.go b/lib/encoder/standard.go index a0f32eb8d..f903d8f22 100644 --- a/lib/encoder/standard.go +++ b/lib/encoder/standard.go @@ -7,9 +7,16 @@ package encoder // List of replaced characters: // (0x00) -> '␀' // SYMBOL FOR NULL // / (slash) -> '/' // FULLWIDTH SOLIDUS -const Standard = MultiEncoder( - EncodeZero | - EncodeSlash | - EncodeCtl | - EncodeDel | - EncodeDot) +const Standard = (EncodeZero | + EncodeSlash | + EncodeCtl | + EncodeDel | + EncodeDot) + +// Base only encodes the zero byte and slash +const Base = (EncodeZero | + EncodeSlash | + EncodeDot) + +// Display is the internal encoding for logging and output +const Display = Standard