drive: rewrite mime type and extension handling

Make use of the mime package to find matching extensions and mime types.
For simplicity, all extensions are now prefixed with "." to match the
mime package requirements.

Parsed extensions get converted if needed.

backend/drive/drive.go

@@ -13,6 +13,7 @@ import (
 	"io"
 	"io/ioutil"
 	"log"
+	"mime"
 	"net/http"
 	"net/url"
 	"os"
@@ -68,32 +69,44 @@ var (
 		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
 		RedirectURL:  oauthutil.TitleBarRedirectURL,
 	}
-	mimeTypeToExtension = map[string]string{
+	_mimeTypeToExtensionDuplicates = map[string]string{
-		"application/epub+zip":                            "epub",
+		"application/x-vnd.oasis.opendocument.presentation": ".odp",
-		"application/msword":                              "doc",
+		"application/x-vnd.oasis.opendocument.spreadsheet":  ".ods",
-		"application/pdf":                                 "pdf",
+		"application/x-vnd.oasis.opendocument.text":         ".odt",
-		"application/rtf":                                 "rtf",
+		"image/jpg":   ".jpg",
-		"application/vnd.ms-excel":                        "xls",
+		"image/x-bmp": ".bmp",
-		"application/vnd.oasis.opendocument.presentation": "odp",
+		"image/x-png": ".png",
-		"application/vnd.oasis.opendocument.spreadsheet":  "ods",
+		"text/rtf":    ".rtf",
-		"application/vnd.oasis.opendocument.text":         "odt",
+	}
-		"application/vnd.openxmlformats-officedocument.presentationml.presentation": "pptx",
+	_mimeTypeToExtension = map[string]string{
-		"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet":         "xlsx",
+		"application/epub+zip":                            ".epub",
-		"application/vnd.openxmlformats-officedocument.wordprocessingml.document":   "docx",
+		"application/json":                                ".json",
-		"application/x-vnd.oasis.opendocument.spreadsheet":                          "ods",
+		"application/msword":                              ".doc",
-		"application/zip":           "zip",
+		"application/pdf":                                 ".pdf",
-		"image/jpeg":                "jpg",
+		"application/rtf":                                 ".rtf",
-		"image/png":                 "png",
+		"application/vnd.ms-excel":                        ".xls",
-		"image/svg+xml":             "svg",
+		"application/vnd.oasis.opendocument.presentation": ".odp",
-		"text/csv":                  "csv",
+		"application/vnd.oasis.opendocument.spreadsheet":  ".ods",
-		"text/html":                 "html",
+		"application/vnd.oasis.opendocument.text":         ".odt",
-		"text/plain":                "txt",
+		"application/vnd.openxmlformats-officedocument.presentationml.presentation": ".pptx",
-		"text/tab-separated-values": "tsv",
+		"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet":         ".xlsx",
+		"application/vnd.openxmlformats-officedocument.wordprocessingml.document":   ".docx",
+		"application/x-msmetafile":  ".wmf",
+		"application/zip":           ".zip",
+		"image/bmp":                 ".bmp",
+		"image/jpeg":                ".jpg",
+		"image/pjpeg":               ".pjpeg",
+		"image/png":                 ".png",
+		"image/svg+xml":             ".svg",
+		"text/csv":                  ".csv",
+		"text/html":                 ".html",
+		"text/plain":                ".txt",
+		"text/tab-separated-values": ".tsv",
 	}
 	extensionToMimeType map[string]string
 	partialFields       = "id,name,size,md5Checksum,trashed,modifiedTime,createdTime,mimeType,parents"
 	exportFormatsOnce   sync.Once           // make sure we fetch the export formats only once
-	_exportFormats      map[string][]string // allowed export mime-type conversions
+	_exportFormats      map[string][]string // allowed export MIME type conversions
 )
 
 // Register with Fs
@@ -252,10 +265,15 @@ func init() {
 		}},
 	})
 
-	// Invert mimeTypeToExtension
+	// register duplicate MIME types first
-	extensionToMimeType = make(map[string]string, len(mimeTypeToExtension))
+	// this allows them to be used with mime.ExtensionsByType() but
-	for mimeType, extension := range mimeTypeToExtension {
+	// mime.TypeByExtension() will return the later registered MIME type
-		extensionToMimeType[extension] = mimeType
+	for _, m := range []map[string]string{_mimeTypeToExtensionDuplicates, _mimeTypeToExtension} {
+		for mimeType, extension := range m {
+			if err := mime.AddExtensionType(extension, mimeType); err != nil {
+				log.Fatalf("Failed to register MIME type %q: %v", mimeType, err)
+			}
+		}
 	}
 }
 
@@ -426,7 +444,7 @@ func (f *Fs) list(dirIDs []string, title string, directoriesOnly bool, filesOnly
 		// if the search title contains an extension and the extension is in the export extensions add a search
 		// for the filename without the extension.
 		// assume that export extensions don't contain escape sequences and only have one part (not .tar.gz)
-		if ext := path.Ext(searchTitle); handleGdocs && len(ext) > 0 && containsString(f.extensions, ext[1:]) {
+		if ext := path.Ext(searchTitle); handleGdocs && len(ext) > 0 && containsString(f.extensions, ext) {
 			stem = title[:len(title)-len(ext)]
 			query = append(query, fmt.Sprintf("(name='%s' or name='%s')", searchTitle, searchTitle[:len(searchTitle)-len(ext)]))
 		} else {
@@ -516,25 +534,78 @@ func isPowerOfTwo(x int64) bool {
 	}
 }
 
-// parseExtensions parses drive export extensions from a string
+// add a charset parameter to all text/* MIME types
-func (f *Fs) parseExtensions(extensions string) error {
+func fixMimeType(mimeType string) string {
-	for _, extension := range strings.Split(extensions, ",") {
+	mediaType, param, err := mime.ParseMediaType(mimeType)
-		extension = strings.ToLower(strings.TrimSpace(extension))
+	if err != nil {
-		if _, found := extensionToMimeType[extension]; !found {
+		return mimeType
-			return errors.Errorf("couldn't find mime type for extension %q", extension)
+	}
-		}
+	if strings.HasPrefix(mimeType, "text/") && param["charset"] == "" {
-		found := false
+		param["charset"] = "utf-8"
-		for _, existingExtension := range f.extensions {
+		mimeType = mime.FormatMediaType(mediaType, param)
-			if extension == existingExtension {
+	}
-				found = true
+	return mimeType
-				break
+}
+func fixMimeTypeMap(m map[string][]string) map[string][]string {
+	for _, v := range m {
+		for i, mt := range v {
+			fixed := fixMimeType(mt)
+			if fixed == "" {
+				panic(errors.Errorf("unable to fix MIME type %q", mt))
 			}
-		}
+			v[i] = fixed
-		if !found {
-			f.extensions = append(f.extensions, extension)
 		}
 	}
-	return nil
+	return m
+}
+func isInternalMimeType(mimeType string) bool {
+	return strings.HasPrefix(mimeType, "application/vnd.google-apps.")
+}
+
+// parseExtensions parses a list of comma separated extensions
+// into a list of unique extensions with leading "."
+func parseExtensions(extensions ...string) ([]string, error) {
+	var result []string
+	for _, extensionText := range extensions {
+		for _, extension := range strings.Split(extensionText, ",") {
+			extension = strings.ToLower(strings.TrimSpace(extension))
+			if len(extension) > 0 && extension[0] != '.' {
+				extension = "." + extension
+			}
+			if mime.TypeByExtension(extension) == "" {
+				return result, errors.Errorf("couldn't find MIME type for extension %q", extension)
+			}
+			found := false
+			for _, existingExtension := range result {
+				if extension == existingExtension {
+					found = true
+					break
+				}
+			}
+			if !found {
+				result = append(result, extension)
+			}
+		}
+	}
+	return result, nil
+}
+
+// parseExtensionMimeTypes parses the given extensions using parseExtensions
+// and maps each resulting extension to its MIME type.
+func parseExtensionMimeTypes(extensions ...string) ([]string, error) {
+	parsedExtensions, err := parseExtensions(extensions...)
+	if err != nil {
+		return nil, err
+	}
+	mimeTypes := make([]string, 0, len(parsedExtensions))
+	for i, extension := range parsedExtensions {
+		mt := mime.TypeByExtension(extension)
+		if mt == "" {
+			return nil, errors.Errorf("couldn't find MIME type for extension %q", extension)
+		}
+		mimeTypes[i] = mt
+	}
+	return mimeTypes, nil
 }
 
 // Figure out if the user wants to use a team drive
@@ -699,11 +770,7 @@ func NewFs(name, path string, m configmap.Mapper) (fs.Fs, error) {
 	f.dirCache = dircache.New(root, f.rootFolderID, f)
 
 	// Parse extensions
-	err = f.parseExtensions(opt.Extensions)
+	f.extensions, err = parseExtensions(opt.Extensions, defaultExtensions)
-	if err != nil {
-		return nil, err
-	}
-	err = f.parseExtensions(defaultExtensions) // make sure there are some sensible ones on there
 	if err != nil {
 		return nil, err
 	}
@@ -836,12 +903,12 @@ func (f *Fs) exportFormats() map[string][]string {
 			_exportFormats = map[string][]string{}
 			return
 		}
-		_exportFormats = about.ExportFormats
+		_exportFormats = fixMimeTypeMap(about.ExportFormats)
 	})
 	return _exportFormats
 }
 
-// findExportFormat works out the optimum extension and mime-type
+// findExportFormat works out the optimum extension and MIME type
 // for this item.
 //
 // Look through the extensions and find the first format that can be
@@ -849,11 +916,11 @@ func (f *Fs) exportFormats() map[string][]string {
 func (f *Fs) findExportFormat(item *drive.File) (extension, filename, mimeType string, isDocument bool) {
 	exportMimeTypes, isDocument := f.exportFormats()[item.MimeType]
 	if isDocument {
-		for _, extension := range f.extensions {
+		for _, _extension := range f.extensions {
-			mimeType := extensionToMimeType[extension]
+			_mimeType := mime.TypeByExtension(_extension)
 			for _, emt := range exportMimeTypes {
-				if emt == mimeType {
+				if emt == _mimeType {
-					return extension, fmt.Sprintf("%s.%s", item.Name, extension), mimeType, true
+					return _extension, item.Name + _extension, _mimeType, true
 				}
 			}
 		}
@@ -1103,27 +1170,11 @@ func (f *Fs) itemToDirEntry(remote string, item *drive.File) (fs.DirEntry, error
 			fs.Debugf(remote, "No export formats found for %q", item.MimeType)
 			break
 		}
-		o, err := f.newObjectWithInfo(remote+"."+extension, item)
+		o, err := f.newObjectWithInfo(remote+extension, item)
 		if err != nil {
 			return nil, err
 		}
-		obj := o.(*Object)
+		o.(*Object).setGdocsMetaData(item, extension, exportMimeType)
-		obj.url = fmt.Sprintf("%sfiles/%s/export?mimeType=%s", f.svc.BasePath, item.Id, url.QueryEscape(exportMimeType))
-		if f.opt.AlternateExport {
-			switch item.MimeType {
-			case "application/vnd.google-apps.drawing":
-				obj.url = fmt.Sprintf("https://docs.google.com/drawings/d/%s/export/%s", item.Id, extension)
-			case "application/vnd.google-apps.document":
-				obj.url = fmt.Sprintf("https://docs.google.com/document/d/%s/export?format=%s", item.Id, extension)
-			case "application/vnd.google-apps.spreadsheet":
-				obj.url = fmt.Sprintf("https://docs.google.com/spreadsheets/d/%s/export?format=%s", item.Id, extension)
-			case "application/vnd.google-apps.presentation":
-				obj.url = fmt.Sprintf("https://docs.google.com/presentation/d/%s/export/%s", item.Id, extension)
-			}
-		}
-		obj.isDocument = true
-		obj.mimeType = exportMimeType
-		obj.bytes = -1
 		return o, nil
 	}
 	return nil, nil
@@ -1866,13 +1917,13 @@ func (o *Object) setGdocsMetaData(info *drive.File, extension, exportMimeType st
 	if o.fs.opt.AlternateExport {
 		switch info.MimeType {
 		case "application/vnd.google-apps.drawing":
-			o.url = fmt.Sprintf("https://docs.google.com/drawings/d/%s/export/%s", info.Id, extension)
+			o.url = fmt.Sprintf("https://docs.google.com/drawings/d/%s/export/%s", info.Id, extension[1:])
 		case "application/vnd.google-apps.document":
-			o.url = fmt.Sprintf("https://docs.google.com/document/d/%s/export?format=%s", info.Id, extension)
+			o.url = fmt.Sprintf("https://docs.google.com/document/d/%s/export?format=%s", info.Id, extension[1:])
 		case "application/vnd.google-apps.spreadsheet":
-			o.url = fmt.Sprintf("https://docs.google.com/spreadsheets/d/%s/export?format=%s", info.Id, extension)
+			o.url = fmt.Sprintf("https://docs.google.com/spreadsheets/d/%s/export?format=%s", info.Id, extension[1:])
 		case "application/vnd.google-apps.presentation":
-			o.url = fmt.Sprintf("https://docs.google.com/presentation/d/%s/export/%s", info.Id, extension)
+			o.url = fmt.Sprintf("https://docs.google.com/presentation/d/%s/export/%s", info.Id, extension[1:])
 		}
 	}
 	o.isDocument = true

backend/drive/drive_internal_test.go

@@ -2,6 +2,7 @@ package drive
 
 import (
 	"encoding/json"
+	"mime"
 	"testing"
 
 	"google.golang.org/api/drive/v3"
@@ -57,6 +58,7 @@ const exampleExportFormats = `{
 func TestInternalLoadExampleExportFormats(t *testing.T) {
 	exportFormatsOnce.Do(func() {})
 	assert.NoError(t, json.Unmarshal([]byte(exampleExportFormats), &_exportFormats))
+	_exportFormats = fixMimeTypeMap(_exportFormats)
 }
 
 func TestInternalParseExtensions(t *testing.T) {
@@ -65,27 +67,24 @@ func TestInternalParseExtensions(t *testing.T) {
 		want    []string
 		wantErr error
 	}{
-		{"doc", []string{"doc"}, nil},
+		{"doc", []string{".doc"}, nil},
-		{" docx ,XLSX, 	pptx,svg", []string{"docx", "xlsx", "pptx", "svg"}, nil},
+		{" docx ,XLSX, 	pptx,svg", []string{".docx", ".xlsx", ".pptx", ".svg"}, nil},
-		{"docx,svg,Docx", []string{"docx", "svg"}, nil},
+		{"docx,svg,Docx", []string{".docx", ".svg"}, nil},
-		{"docx,potato,docx", []string{"docx"}, errors.New(`couldn't find mime type for extension "potato"`)},
+		{"docx,potato,docx", []string{".docx"}, errors.New(`couldn't find MIME type for extension ".potato"`)},
 	} {
-		f := new(Fs)
+		extensions, gotErr := parseExtensions(test.in)
-		gotErr := f.parseExtensions(test.in)
 		if test.wantErr == nil {
 			assert.NoError(t, gotErr)
 		} else {
 			assert.EqualError(t, gotErr, test.wantErr.Error())
 		}
-		assert.Equal(t, test.want, f.extensions)
+		assert.Equal(t, test.want, extensions)
 	}
 
 	// Test it is appending
-	f := new(Fs)
+	extensions, gotErr := parseExtensions("docx,svg", "docx,svg,xlsx")
-	assert.Nil(t, f.parseExtensions("docx,svg"))
+	assert.NoError(t, gotErr)
-	assert.Nil(t, f.parseExtensions("docx,svg,xlsx"))
+	assert.Equal(t, []string{".docx", ".svg", ".xlsx"}, extensions)
-	assert.Equal(t, []string{"docx", "svg", "xlsx"}, f.extensions)
-
 }
 
 func TestInternalFindExportFormat(t *testing.T) {
@@ -99,10 +98,10 @@ func TestInternalFindExportFormat(t *testing.T) {
 		wantMimeType  string
 	}{
 		{[]string{}, "", ""},
-		{[]string{"pdf"}, "pdf", "application/pdf"},
+		{[]string{".pdf"}, ".pdf", "application/pdf"},
-		{[]string{"pdf", "rtf", "xls"}, "pdf", "application/pdf"},
+		{[]string{".pdf", ".rtf", ".xls"}, ".pdf", "application/pdf"},
-		{[]string{"xls", "rtf", "pdf"}, "rtf", "application/rtf"},
+		{[]string{".xls", ".rtf", ".pdf"}, ".rtf", "application/rtf"},
-		{[]string{"xls", "csv", "svg"}, "", ""},
+		{[]string{".xls", ".csv", ".svg"}, "", ""},
 	} {
 		f := new(Fs)
 		f.extensions = test.extensions
@@ -117,3 +116,35 @@ func TestInternalFindExportFormat(t *testing.T) {
 		assert.Equal(t, true, gotIsDocument)
 	}
 }
+
+func TestMimeTypesToExtension(t *testing.T) {
+	for mimeType, extension := range _mimeTypeToExtension {
+		extensions, err := mime.ExtensionsByType(mimeType)
+		assert.NoError(t, err)
+		assert.Contains(t, extensions, extension)
+	}
+}
+
+func TestExtensionToMimeType(t *testing.T) {
+	for mimeType, extension := range _mimeTypeToExtension {
+		gotMimeType := mime.TypeByExtension(extension)
+		mediatype, _, err := mime.ParseMediaType(gotMimeType)
+		assert.NoError(t, err)
+		assert.Equal(t, mimeType, mediatype)
+	}
+}
+
+func TestExtensionsForExportFormats(t *testing.T) {
+	if _exportFormats == nil {
+		t.Error("exportFormats == nil")
+	}
+	for fromMT, toMTs := range _exportFormats {
+		for _, toMT := range toMTs {
+			if !isInternalMimeType(toMT) {
+				extensions, err := mime.ExtensionsByType(toMT)
+				assert.NoError(t, err, "invalid MIME type %q", toMT)
+				assert.NotEmpty(t, extensions, "No extension found for %q (from: %q)", fromMT, toMT)
+			}
+		}
+	}
+}