community
/
rclone
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fs/hash: align hashsum names and update documentation (#5339) 

- Unify all hash names as lowercase alphanumerics without punctuation.
- Legacy names continue to work but disappear from docs, they can be depreciated or dropped later.
- Make rclone hashsum print supported hash list in case of wrong spelling.
- Update documentation.

Fixes #5071
Fixes #4841
This commit is contained in:
Ivan Andreev 2021-05-21 17:32:33 +03:00 committed by GitHub
parent 07f2f3a62e
commit 5b6f637461
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 122 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
backend/dropbox/dropbox.go
View File

@ -138,7 +138,7 @@ func getOauthConfig(m configmap.Mapper) *oauth2.Config {
// Register with Fs
func init() {
DbHashType = hash.RegisterHash("DropboxHash", 64, dbhash.New)
DbHashType = hash.RegisterHash("dropbox", "DropboxHash", 64, dbhash.New)
fs.Register(&fs.RegInfo{
Name: "dropbox",
Description: "Dropbox",

2
backend/mailru/mailru.go
View File

@ -80,7 +80,7 @@ var oauthConfig = &oauth2.Config{
// Register with Fs
func init() {
MrHashType = hash.RegisterHash("MailruHash", 40, mrhash.New)
MrHashType = hash.RegisterHash("mailru", "MailruHash", 40, mrhash.New)
fs.Register(&fs.RegInfo{
Name: "mailru",
Description: "Mail.ru Cloud",

2
backend/onedrive/onedrive.go
View File

@ -93,7 +93,7 @@ var (
// Register with Fs
func init() {
QuickXorHashType = hash.RegisterHash("QuickXorHash", 40, quickxorhash.New)
QuickXorHashType = hash.RegisterHash("quickxor", "QuickXorHash", 40, quickxorhash.New)
fs.Register(&fs.RegInfo{
Name: "onedrive",
Description: "Microsoft OneDrive",

27
cmd/hashsum/hashsum.go
View File

@ -4,6 +4,7 @@ import (
"context"
"fmt"
"os"
"strings"
"github.com/pkg/errors"
"github.com/rclone/rclone/cmd"
@ -69,23 +70,17 @@ hashed locally enabling any hash for any remote.
Run without a hash to see the list of all supported hashes, e.g.
$ rclone hashsum
Supported hashes are:
* MD5
* SHA-1
* DropboxHash
* QuickXorHash
` + hashListHelp(" ") + `
Then
$ rclone hashsum MD5 remote:path
Note that hash names are case insensitive.
`,
RunE: func(command *cobra.Command, args []string) error {
cmd.CheckArgs(0, 2, command, args)
if len(args) == 0 {
fmt.Printf("Supported hashes are:\n")
for _, ht := range hash.Supported().Array() {
fmt.Printf(" * %v\n", ht)
}
fmt.Print(hashListHelp(""))
return nil
} else if len(args) == 1 {
return errors.New("need hash type and remote")
@ -93,6 +88,7 @@ Then
var ht hash.Type
err := ht.Set(args[0])
if err != nil {
fmt.Println(hashListHelp(""))
return err
}
fsrc := cmd.NewFsSrc(args[1:])
@ -111,3 +107,14 @@ Then
return nil
},
}
func hashListHelp(indent string) string {
var help strings.Builder
help.WriteString(indent)
help.WriteString("Supported hashes are:\n")
for _, ht := range hash.Supported().Array() {
help.WriteString(indent)
fmt.Fprintf(&help, " * %v\n", ht.String())
}
return help.String()
}

13
docs/content/commands/rclone_hashsum.md
View File

@ -25,15 +25,20 @@ Run without a hash to see the list of all supported hashes, e.g.
$ rclone hashsum
Supported hashes are:
* MD5
* SHA-1
* DropboxHash
* QuickXorHash
* md5
* sha1
* whirlpool
* crc32
* dropbox
* mailru
* quickxor
Then
$ rclone hashsum MD5 remote:path
Note that hash names are case insensitive.
```
rclone hashsum <hash> remote:path [flags]

0
docs/content/flags.md Executable file → Normal file
View File

101
fs/hash/hash.go
View File

@ -20,25 +20,36 @@ type Type int
type hashDefinition struct {
width int
name string
alias string
newFunc func() hash.Hash
hashType Type
}
var hashes []*hashDefinition
var highestType Type = 1
var (
type2hash = map[Type]*hashDefinition{}
name2hash = map[string]*hashDefinition{}
alias2hash = map[string]*hashDefinition{}
supported = []Type{}
)
// RegisterHash adds a new Hash to the list and returns it Type
func RegisterHash(name string, width int, newFunc func() hash.Hash) Type {
func RegisterHash(name, alias string, width int, newFunc func() hash.Hash) Type {
hashType := Type(1 << len(supported))
supported = append(supported, hashType)
definition := &hashDefinition{
name: name,
alias: alias,
width: width,
newFunc: newFunc,
hashType: highestType,
hashType: hashType,
}
hashes = append(hashes, definition)
highestType = highestType << 1
return definition.hashType
type2hash[hashType] = definition
name2hash[name] = definition
alias2hash[alias] = definition
return hashType
}
// ErrUnsupported should be returned by filesystem,
@ -63,31 +74,23 @@ var (
)
func init() {
MD5 = RegisterHash("MD5", 32, md5.New)
SHA1 = RegisterHash("SHA-1", 40, sha1.New)
Whirlpool = RegisterHash("Whirlpool", 128, whirlpool.New)
CRC32 = RegisterHash("CRC-32", 8, func() hash.Hash { return crc32.NewIEEE() })
MD5 = RegisterHash("md5", "MD5", 32, md5.New)
SHA1 = RegisterHash("sha1", "SHA-1", 40, sha1.New)
Whirlpool = RegisterHash("whirlpool", "Whirlpool", 128, whirlpool.New)
CRC32 = RegisterHash("crc32", "CRC-32", 8, func() hash.Hash { return crc32.NewIEEE() })
}
// Supported returns a set of all the supported hashes by
// HashStream and MultiHasher.
func Supported() Set {
var types []Type
for _, v := range hashes {
types = append(types, v.hashType)
}
return NewHashSet(types...)
return NewHashSet(supported...)
}
// Width returns the width in characters for any HashType
func Width(hashType Type) int {
for _, v := range hashes {
if v.hashType == hashType {
return v.width
}
if hash := type2hash[hashType]; hash != nil {
return hash.width
}
return 0
}
@ -118,32 +121,29 @@ func StreamTypes(r io.Reader, set Set) (map[Type]string, error) {
// The function will panic if the hash type is unknown.
func (h Type) String() string {
if h == None {
return "None"
return "none"
}
for _, v := range hashes {
if v.hashType == h {
return v.name
}
if hash := type2hash[h]; hash != nil {
return hash.name
}
err := fmt.Sprintf("internal error: unknown hash type: 0x%x", int(h))
panic(err)
panic(fmt.Sprintf("internal error: unknown hash type: 0x%x", int(h)))
}
// Set a Type from a flag
// Set a Type from a flag.
// Both name and alias are accepted.
func (h *Type) Set(s string) error {
if s == "None" {
if s == "none" || s == "None" {
*h = None
return nil
}
for _, v := range hashes {
if v.name == s {
*h = v.hashType
return nil
}
if hash := name2hash[strings.ToLower(s)]; hash != nil {
*h = hash.hashType
return nil
}
if hash := alias2hash[s]; hash != nil {
*h = hash.hashType
return nil
}
return errors.Errorf("Unknown hash type %q", s)
}
@ -159,23 +159,14 @@ func fromTypes(set Set) (map[Type]hash.Hash, error) {
if !set.SubsetOf(Supported()) {
return nil, errors.Errorf("requested set %08x contains unknown hash types", int(set))
}
var hashers = make(map[Type]hash.Hash)
hashers := map[Type]hash.Hash{}
types := set.Array()
for _, t := range types {
for _, v := range hashes {
if t != v.hashType {
continue
}
hashers[t] = v.newFunc()
break
}
if hashers[t] == nil {
err := fmt.Sprintf("internal error: Unsupported hash type %v", t)
panic(err)
for _, t := range set.Array() {
hash := type2hash[t]
if hash == nil {
panic(fmt.Sprintf("internal error: Unsupported hash type %v", t))
}
hashers[t] = hash.newFunc()
}
return hashers, nil

47
fs/hash/hash_test.go
View File

@ -156,16 +156,53 @@ func TestHashStreamTypes(t *testing.T) {
func TestHashSetStringer(t *testing.T) {
h := hash.NewHashSet(hash.SHA1, hash.MD5)
assert.Equal(t, h.String(), "[MD5, SHA-1]")
assert.Equal(t, "[md5, sha1]", h.String())
h = hash.NewHashSet(hash.SHA1)
assert.Equal(t, h.String(), "[SHA-1]")
assert.Equal(t, "[sha1]", h.String())
h = hash.NewHashSet()
assert.Equal(t, h.String(), "[]")
assert.Equal(t, "[]", h.String())
}
func TestHashStringer(t *testing.T) {
h := hash.MD5
assert.Equal(t, h.String(), "MD5")
assert.Equal(t, "md5", h.String())
h = hash.SHA1
assert.Equal(t, "sha1", h.String())
h = hash.None
assert.Equal(t, h.String(), "None")
assert.Equal(t, "none", h.String())
}
func TestHashSetter(t *testing.T) {
var ht hash.Type
assert.NoError(t, ht.Set("none"))
assert.Equal(t, hash.None, ht)
assert.NoError(t, ht.Set("None"))
assert.Equal(t, hash.None, ht)
assert.NoError(t, ht.Set("md5"))
assert.Equal(t, hash.MD5, ht)
assert.NoError(t, ht.Set("MD5"))
assert.Equal(t, hash.MD5, ht)
assert.NoError(t, ht.Set("sha1"))
assert.Equal(t, hash.SHA1, ht)
assert.NoError(t, ht.Set("SHA-1"))
assert.Equal(t, hash.SHA1, ht)
assert.NoError(t, ht.Set("SHA1"))
assert.Equal(t, hash.SHA1, ht)
assert.NoError(t, ht.Set("Sha1"))
assert.Equal(t, hash.SHA1, ht)
assert.Error(t, ht.Set("Sha-1"))
}
func TestHashTypeStability(t *testing.T) {
assert.Equal(t, hash.Type(0), hash.None)
assert.Equal(t, hash.Type(1), hash.MD5)
assert.Equal(t, hash.Type(2), hash.SHA1)
assert.True(t, hash.Supported().Contains(hash.MD5))
assert.True(t, hash.Supported().Contains(hash.SHA1))
assert.False(t, hash.Supported().Contains(hash.None))
}

8
fs/operations/operations_test.go
View File

@ -1206,10 +1206,10 @@ func TestListFormat(t *testing.T) {
Format: "2006-01-02T15:04:05.000000000Z07:00"},
IsDir: false,
Hashes: map[string]string{
"MD5": "0cc175b9c0f1b6a831c399e269772661",
"SHA-1": "86f7e437faa5a7fce15d1ddcb9eaeaea377667b8",
"DropboxHash": "bf5d3affb73efd2ec6c36ad3112dd933efed63c4e1cbffcfa88e2759c144f2d8",
"QuickXorHash": "6100000000000000000000000100000000000000"},
"md5": "0cc175b9c0f1b6a831c399e269772661",
"sha1": "86f7e437faa5a7fce15d1ddcb9eaeaea377667b8",
"dropbox": "bf5d3affb73efd2ec6c36ad3112dd933efed63c4e1cbffcfa88e2759c144f2d8",
"quickxor": "6100000000000000000000000100000000000000"},
ID: "fileID",
OrigID: "fileOrigID",
}

2
fs/options_test.go
View File

@ -115,7 +115,7 @@ func TestHTTPOption(t *testing.T) {
func TestHashesOption(t *testing.T) {
opt := &HashesOption{hash.Set(hash.MD5 | hash.SHA1)}
var _ OpenOption = opt // check interface
assert.Equal(t, `HashesOption([MD5, SHA-1])`, opt.String())
assert.Equal(t, `HashesOption([md5, sha1])`, opt.String())
key, value := opt.Header()
assert.Equal(t, "", key)
assert.Equal(t, "", value)