Add generic content digest tool

Previously a useful gist, this changeset polishes the original tarsum tool into
a utility that can be used to calculate content digests. Any algorithm from the
digest package is supported with additional support from tarsum.

This tool is very useful for quickly checking backend digests and verifying
correctness.

Signed-off-by: Stephen J Day <stephen.day@docker.com>

Makefile

@@ -28,6 +28,10 @@ ${PREFIX}/bin/registry: version/version.go $(shell find . -type f -name '*.go')
 	@echo "+ $@"
 	@go build -tags "${DOCKER_BUILDTAGS}" -o $@ ${GO_LDFLAGS}  ${GO_GCFLAGS} ./cmd/registry
 
+${PREFIX}/bin/digest: version/version.go $(shell find . -type f -name '*.go')
+	@echo "+ $@"
+	@go build -tags "${DOCKER_BUILDTAGS}" -o $@ ${GO_LDFLAGS}  ${GO_GCFLAGS} ./cmd/digest
+
 ${PREFIX}/bin/registry-api-descriptor-template: version/version.go $(shell find . -type f -name '*.go')
 	@echo "+ $@"
 	@go build -o $@ ${GO_LDFLAGS} ${GO_GCFLAGS} ./cmd/registry-api-descriptor-template
@@ -62,7 +66,7 @@ test-full:
 	@echo "+ $@"
 	@go test ./...
 
-binaries: ${PREFIX}/bin/registry ${PREFIX}/bin/registry-api-descriptor-template
+binaries: ${PREFIX}/bin/registry ${PREFIX}/bin/digest ${PREFIX}/bin/registry-api-descriptor-template
 	@echo "+ $@"
 
 clean:

cmd/digest/main.go

@@ -0,0 +1,129 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"strings"
+
+	"github.com/docker/distribution/digest"
+	"github.com/docker/distribution/version"
+	"github.com/docker/docker/pkg/tarsum"
+)
+
+var (
+	algorithm   = digest.Canonical
+	showVersion bool
+)
+
+type job struct {
+	name   string
+	reader io.Reader
+}
+
+func init() {
+	flag.Var(&algorithm, "a", "select the digest algorithm (shorthand)")
+	flag.Var(&algorithm, "algorithm", "select the digest algorithm")
+	flag.BoolVar(&showVersion, "version", false, "show the version and exit")
+
+	log.SetFlags(0)
+	log.SetPrefix(os.Args[0] + ": ")
+}
+
+func usage() {
+	fmt.Fprintf(os.Stderr, "usage: %s [files...]\n", os.Args[0])
+	fmt.Fprintf(os.Stderr, `
+Calculate the digest of one or more input files, emitting the result
+to standard out. If no files are provided, the digest of stdin will
+be calculated.
+
+`)
+	flag.PrintDefaults()
+}
+
+func unsupported() {
+	log.Fatalf("unsupported digest algorithm: %v", algorithm)
+}
+
+func main() {
+	var jobs []job
+
+	flag.Usage = usage
+	flag.Parse()
+	if showVersion {
+		version.PrintVersion()
+		return
+	}
+
+	var fail bool // if we fail on one item, foul the exit code
+	if flag.NArg() > 0 {
+		for _, path := range flag.Args() {
+			fp, err := os.Open(path)
+
+			if err != nil {
+				log.Printf("%s: %v", path, err)
+				fail = true
+				continue
+			}
+			defer fp.Close()
+
+			jobs = append(jobs, job{name: path, reader: fp})
+		}
+	} else {
+		// just read stdin
+		jobs = append(jobs, job{name: "-", reader: os.Stdin})
+	}
+
+	digestFn := algorithm.FromReader
+
+	if !algorithm.Available() {
+		// we cannot digest if is not available. An exception is made for
+		// tarsum.
+		if !strings.HasPrefix(algorithm.String(), "tarsum") {
+			unsupported()
+		}
+
+		var version tarsum.Version
+		if algorithm == "tarsum" {
+			// small hack: if we just have tarsum, use latest
+			version = tarsum.Version1
+		} else {
+			var err error
+			version, err = tarsum.GetVersionFromTarsum(algorithm.String())
+			if err != nil {
+				unsupported()
+			}
+		}
+
+		digestFn = func(rd io.Reader) (digest.Digest, error) {
+			ts, err := tarsum.NewTarSum(rd, true, version)
+			if err != nil {
+				return "", err
+			}
+
+			if _, err := io.Copy(ioutil.Discard, ts); err != nil {
+				return "", err
+			}
+
+			return digest.Digest(ts.Sum(nil)), nil
+		}
+	}
+
+	for _, job := range jobs {
+		dgst, err := digestFn(job.reader)
+		if err != nil {
+			log.Printf("%s: %v", job.name, err)
+			fail = true
+			continue
+		}
+
+		fmt.Printf("%v\t%s\n", dgst, job.name)
+	}
+
+	if fail {
+		os.Exit(1)
+	}
+}

digest/digest.go

@@ -70,15 +70,10 @@ func ParseDigest(s string) (Digest, error) {
 	return d, d.Validate()
 }
 
-// FromReader returns the most valid digest for the underlying content.
+// FromReader returns the most valid digest for the underlying content using
+// the canonical digest algorithm.
 func FromReader(rd io.Reader) (Digest, error) {
-	digester := Canonical.New()
-
-	if _, err := io.Copy(digester.Hash(), rd); err != nil {
-		return "", err
-	}
-
-	return digester.Digest(), nil
+	return Canonical.FromReader(rd)
 }
 
 // FromTarArchive produces a tarsum digest from reader rd.

digest/digester.go

@@ -3,6 +3,7 @@ package digest
 import (
 	"crypto"
 	"hash"
+	"io"
 )
 
 // Algorithm identifies and implementation of a digester by an identifier.
@@ -49,6 +50,22 @@ func (a Algorithm) Available() bool {
 	return h.Available()
 }
 
+func (a Algorithm) String() string {
+	return string(a)
+}
+
+// Set implemented to allow use of Algorithm as a command line flag.
+func (a *Algorithm) Set(value string) error {
+	if value == "" {
+		*a = Canonical
+	} else {
+		// just do a type conversion, support is queried with Available.
+		*a = Algorithm(value)
+	}
+
+	return nil
+}
+
 // New returns a new digester for the specified algorithm. If the algorithm
 // does not have a digester implementation, nil will be returned. This can be
 // checked by calling Available before calling New.
@@ -69,6 +86,17 @@ func (a Algorithm) Hash() hash.Hash {
 	return algorithms[a].New()
 }
 
+// FromReader returns the digest of the reader using the algorithm.
+func (a Algorithm) FromReader(rd io.Reader) (Digest, error) {
+	digester := a.New()
+
+	if _, err := io.Copy(digester.Hash(), rd); err != nil {
+		return "", err
+	}
+
+	return digester.Digest(), nil
+}
+
 // TODO(stevvooe): Allow resolution of verifiers using the digest type and
 // this registration system.