drive: request the export formats only when required #320

If the listing has no google docs in or the user uses
`--drive-skip-gdocs` then we don't fetch the export formats which
saves a transaction to drive.

backend/drive/drive.go

@@ -19,6 +19,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/ncw/rclone/fs"
@@ -97,6 +98,8 @@ var (
 	}
 	extensionToMimeType map[string]string
 	partialFields       = "id,name,size,md5Checksum,trashed,modifiedTime,mimeType"
+	exportFormatsOnce   sync.Once           // make sure we fetch the export formats only once
+	_exportFormats      map[string][]string // allowed export mime-type conversions
 )
 
 // Register with Fs
@@ -185,7 +188,6 @@ type Fs struct {
 	dirCache     *dircache.DirCache // Map of directory path to directory id
 	pacer        *pacer.Pacer       // To pace the API calls
 	extensions   []string           // preferred extensions to download docs
-	exportFormats map[string][]string // allowed export mime-type conversions
 	teamDriveID  string             // team drive ID, may be ""
 	isTeamDrive  bool               // true if this is a team drive
 }
@@ -530,16 +532,6 @@ func NewFs(name, path string) (fs.Fs, error) {
 
 	f.dirCache = dircache.New(root, f.rootFolderID, f)
 
-	var about *drive.About
-	err = f.pacer.Call(func() (bool, error) {
-		about, err = f.svc.About.Get().Fields("exportFormats").Do()
-		return shouldRetry(err)
-	})
-	if err != nil {
-		return nil, errors.Wrap(err, "couldn't get Drive exportFormats")
-	}
-	f.exportFormats = about.ExportFormats
-
 	// Parse extensions
 	err = f.parseExtensions(*driveExtensions)
 	if err != nil {
@@ -651,6 +643,28 @@ func isAuthOwned(item *drive.File) bool {
 	return false
 }
 
+// exportFormats returns the export formats from drive, fetching them
+// if necessary.
+//
+// if the fetch fails then it will not export any drive formats
+func (f *Fs) exportFormats() map[string][]string {
+	exportFormatsOnce.Do(func() {
+		var about *drive.About
+		var err error
+		err = f.pacer.Call(func() (bool, error) {
+			about, err = f.svc.About.Get().Fields("exportFormats").Do()
+			return shouldRetry(err)
+		})
+		if err != nil {
+			fs.Errorf(f, "Failed to get Drive exportFormats: %v", err)
+			_exportFormats = map[string][]string{}
+			return
+		}
+		_exportFormats = about.ExportFormats
+	})
+	return _exportFormats
+}
+
 // findExportFormat works out the optimum extension and mime-type
 // for this item.
 //
@@ -692,7 +706,6 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 	var iErr error
 	_, err = f.list(directoryID, "", false, false, false, func(item *drive.File) bool {
 		remote := path.Join(dir, item.Name)
-		exportMimeTypes, isDocument := f.exportFormats[item.MimeType]
 		switch {
 		case *driveAuthOwnerOnly && !isAuthOwned(item):
 			// ignore object or directory
@@ -710,30 +723,31 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 				return true
 			}
 			entries = append(entries, o)
-		case isDocument:
+		case *driveSkipGdocs:
+			fs.Debugf(remote, "Skipping google document type %q", item.MimeType)
+		default:
+			exportMimeTypes, isDocument := f.exportFormats()[item.MimeType]
+			if !isDocument {
+				fs.Debugf(remote, "Ignoring unknown document type %q", item.MimeType)
+				break
+			}
 			// If item has export links then it is a google doc
 			extension, exportMimeType := f.findExportFormat(remote, exportMimeTypes)
 			if extension == "" {
-				fs.Debugf(remote, "No export formats found")
+				fs.Debugf(remote, "No export formats found for %q", item.MimeType)
-			} else {
+				break
+			}
 			o, err := f.newObjectWithInfo(remote+"."+extension, item)
 			if err != nil {
 				iErr = err
 				return true
 			}
-				if !*driveSkipGdocs {
 			obj := o.(*Object)
 			obj.url = fmt.Sprintf("%sfiles/%s/export?mimeType=%s", f.svc.BasePath, item.Id, url.QueryEscape(exportMimeType))
 			obj.isDocument = true
 			obj.mimeType = exportMimeType
 			obj.bytes = -1
 			entries = append(entries, o)
-				} else {
-					fs.Debugf(f, "Skip google document: %q", remote)
-				}
-			}
-		default:
-			fs.Debugf(remote, "Ignoring unknown object")
 		}
 		return false
 	})

backend/drive/drive_internal_test.go

@@ -10,7 +10,7 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-const exportFormats = `{
+const exampleExportFormats = `{
 	"application/vnd.google-apps.document": [
 		"application/rtf",
 		"application/vnd.oasis.opendocument.text",
@@ -51,7 +51,14 @@ const exportFormats = `{
 		"application/pdf",
 		"image/jpeg"
 	]
- }`
+}`
+
+var exportFormats map[string][]string
+
+// Load the example export formats into exportFormats for testing
+func TestInternalLoadExampleExportFormats(t *testing.T) {
+	assert.NoError(t, json.Unmarshal([]byte(exampleExportFormats), &exportFormats))
+}
 
 func TestInternalParseExtensions(t *testing.T) {
 	for _, test := range []struct {
@@ -98,8 +105,7 @@ func TestInternalFindExportFormat(t *testing.T) {
 	} {
 		f := new(Fs)
 		f.extensions = test.extensions
-		assert.NoError(t, json.Unmarshal([]byte(exportFormats), &f.exportFormats))
+		gotExtension, gotMimeType := f.findExportFormat("file", exportFormats[item.MimeType])
-		gotExtension, gotMimeType := f.findExportFormat("file", f.exportFormats[item.MimeType])
 		assert.Equal(t, test.wantExtension, gotExtension)
 		assert.Equal(t, test.wantMimeType, gotMimeType)
 	}