community
/
rclone
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

drive: request the export formats only when required #320 

If the listing has no google docs in or the user uses
`--drive-skip-gdocs` then we don't fetch the export formats which
saves a transaction to drive.
This commit is contained in:
Nick Craig-Wood 2018-01-31 20:03:02 +00:00
parent 3c7a755631
commit a6227f34e2
2 changed files with 107 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

100
backend/drive/drive.go
View File

@ -19,6 +19,7 @@ import (
"sort" "sort"
"strconv" "strconv"
"strings" "strings"
"sync"
"time" "time"
"github.com/ncw/rclone/fs" "github.com/ncw/rclone/fs"
@ -97,6 +98,8 @@ var (
} }
extensionToMimeType map[string]string extensionToMimeType map[string]string
partialFields = "id,name,size,md5Checksum,trashed,modifiedTime,mimeType" partialFields = "id,name,size,md5Checksum,trashed,modifiedTime,mimeType"
exportFormatsOnce sync.Once // make sure we fetch the export formats only once
_exportFormats map[string][]string // allowed export mime-type conversions
) )
// Register with Fs // Register with Fs
@ -176,18 +179,17 @@ func init() {
// Fs represents a remote drive server // Fs represents a remote drive server
type Fs struct { type Fs struct {
name string // name of this remote name string // name of this remote
root string // the path we are working on root string // the path we are working on
features *fs.Features // optional features features *fs.Features // optional features
svc *drive.Service // the connection to the drive server svc *drive.Service // the connection to the drive server
client *http.Client // authorized client client *http.Client // authorized client
rootFolderID string // the id of the root folder rootFolderID string // the id of the root folder
dirCache *dircache.DirCache // Map of directory path to directory id dirCache *dircache.DirCache // Map of directory path to directory id
pacer *pacer.Pacer // To pace the API calls pacer *pacer.Pacer // To pace the API calls
extensions []string // preferred extensions to download docs extensions []string // preferred extensions to download docs
exportFormats map[string][]string // allowed export mime-type conversions teamDriveID string // team drive ID, may be ""
teamDriveID string // team drive ID, may be "" isTeamDrive bool // true if this is a team drive
isTeamDrive bool // true if this is a team drive
} }
// Object describes a drive object // Object describes a drive object
@ -530,16 +532,6 @@ func NewFs(name, path string) (fs.Fs, error) {
f.dirCache = dircache.New(root, f.rootFolderID, f) f.dirCache = dircache.New(root, f.rootFolderID, f)
var about *drive.About
err = f.pacer.Call(func() (bool, error) {
about, err = f.svc.About.Get().Fields("exportFormats").Do()
return shouldRetry(err)
})
if err != nil {
return nil, errors.Wrap(err, "couldn't get Drive exportFormats")
}
f.exportFormats = about.ExportFormats
// Parse extensions // Parse extensions
err = f.parseExtensions(*driveExtensions) err = f.parseExtensions(*driveExtensions)
if err != nil { if err != nil {
@ -651,6 +643,28 @@ func isAuthOwned(item *drive.File) bool {
return false return false
} }
// exportFormats returns the export formats from drive, fetching them
// if necessary.
//
// if the fetch fails then it will not export any drive formats
func (f *Fs) exportFormats() map[string][]string {
exportFormatsOnce.Do(func() {
var about *drive.About
var err error
err = f.pacer.Call(func() (bool, error) {
about, err = f.svc.About.Get().Fields("exportFormats").Do()
return shouldRetry(err)
})
if err != nil {
fs.Errorf(f, "Failed to get Drive exportFormats: %v", err)
_exportFormats = map[string][]string{}
return
}
_exportFormats = about.ExportFormats
})
return _exportFormats
}
// findExportFormat works out the optimum extension and mime-type // findExportFormat works out the optimum extension and mime-type
// for this item. // for this item.
// //
@ -692,7 +706,6 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
var iErr error var iErr error
_, err = f.list(directoryID, "", false, false, false, func(item *drive.File) bool { _, err = f.list(directoryID, "", false, false, false, func(item *drive.File) bool {
remote := path.Join(dir, item.Name) remote := path.Join(dir, item.Name)
exportMimeTypes, isDocument := f.exportFormats[item.MimeType]
switch { switch {
case *driveAuthOwnerOnly && !isAuthOwned(item): case *driveAuthOwnerOnly && !isAuthOwned(item):
// ignore object or directory // ignore object or directory
@ -710,30 +723,31 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
return true return true
} }
entries = append(entries, o) entries = append(entries, o)
case isDocument: case *driveSkipGdocs:
fs.Debugf(remote, "Skipping google document type %q", item.MimeType)
default:
exportMimeTypes, isDocument := f.exportFormats()[item.MimeType]
if !isDocument {
fs.Debugf(remote, "Ignoring unknown document type %q", item.MimeType)
break
}
// If item has export links then it is a google doc // If item has export links then it is a google doc
extension, exportMimeType := f.findExportFormat(remote, exportMimeTypes) extension, exportMimeType := f.findExportFormat(remote, exportMimeTypes)
if extension == "" { if extension == "" {
fs.Debugf(remote, "No export formats found") fs.Debugf(remote, "No export formats found for %q", item.MimeType)
} else { break
o, err := f.newObjectWithInfo(remote+"."+extension, item)
if err != nil {
iErr = err
return true
}
if !*driveSkipGdocs {
obj := o.(*Object)
obj.url = fmt.Sprintf("%sfiles/%s/export?mimeType=%s", f.svc.BasePath, item.Id, url.QueryEscape(exportMimeType))
obj.isDocument = true
obj.mimeType = exportMimeType
obj.bytes = -1
entries = append(entries, o)
} else {
fs.Debugf(f, "Skip google document: %q", remote)
}
} }
default: o, err := f.newObjectWithInfo(remote+"."+extension, item)
fs.Debugf(remote, "Ignoring unknown object") if err != nil {
iErr = err
return true
}
obj := o.(*Object)
obj.url = fmt.Sprintf("%sfiles/%s/export?mimeType=%s", f.svc.BasePath, item.Id, url.QueryEscape(exportMimeType))
obj.isDocument = true
obj.mimeType = exportMimeType
obj.bytes = -1
entries = append(entries, o)
} }
return false return false
}) })

94
backend/drive/drive_internal_test.go
View File

@ -10,48 +10,55 @@ import (
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
) )
const exportFormats = `{ const exampleExportFormats = `{
"application/vnd.google-apps.document": [ "application/vnd.google-apps.document": [
"application/rtf", "application/rtf",
"application/vnd.oasis.opendocument.text", "application/vnd.oasis.opendocument.text",
"text/html", "text/html",
"application/pdf", "application/pdf",
"application/epub+zip", "application/epub+zip",
"application/zip", "application/zip",
"application/vnd.openxmlformats-officedocument.wordprocessingml.document", "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
"text/plain" "text/plain"
], ],
"application/vnd.google-apps.spreadsheet": [ "application/vnd.google-apps.spreadsheet": [
"application/x-vnd.oasis.opendocument.spreadsheet", "application/x-vnd.oasis.opendocument.spreadsheet",
"text/tab-separated-values", "text/tab-separated-values",
"application/pdf", "application/pdf",
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"text/csv", "text/csv",
"application/zip", "application/zip",
"application/vnd.oasis.opendocument.spreadsheet" "application/vnd.oasis.opendocument.spreadsheet"
], ],
"application/vnd.google-apps.jam": [ "application/vnd.google-apps.jam": [
"application/pdf" "application/pdf"
], ],
"application/vnd.google-apps.script": [ "application/vnd.google-apps.script": [
"application/vnd.google-apps.script+json" "application/vnd.google-apps.script+json"
], ],
"application/vnd.google-apps.presentation": [ "application/vnd.google-apps.presentation": [
"application/vnd.oasis.opendocument.presentation", "application/vnd.oasis.opendocument.presentation",
"application/pdf", "application/pdf",
"application/vnd.openxmlformats-officedocument.presentationml.presentation", "application/vnd.openxmlformats-officedocument.presentationml.presentation",
"text/plain" "text/plain"
], ],
"application/vnd.google-apps.form": [ "application/vnd.google-apps.form": [
"application/zip" "application/zip"
], ],
"application/vnd.google-apps.drawing": [ "application/vnd.google-apps.drawing": [
"image/svg+xml", "image/svg+xml",
"image/png", "image/png",
"application/pdf", "application/pdf",
"image/jpeg" "image/jpeg"
] ]
}` }`
var exportFormats map[string][]string
// Load the example export formats into exportFormats for testing
func TestInternalLoadExampleExportFormats(t *testing.T) {
assert.NoError(t, json.Unmarshal([]byte(exampleExportFormats), &exportFormats))
}
func TestInternalParseExtensions(t *testing.T) { func TestInternalParseExtensions(t *testing.T) {
for _, test := range []struct { for _, test := range []struct {
@ -98,8 +105,7 @@ func TestInternalFindExportFormat(t *testing.T) {
} { } {
f := new(Fs) f := new(Fs)
f.extensions = test.extensions f.extensions = test.extensions
assert.NoError(t, json.Unmarshal([]byte(exportFormats), &f.exportFormats)) gotExtension, gotMimeType := f.findExportFormat("file", exportFormats[item.MimeType])
gotExtension, gotMimeType := f.findExportFormat("file", f.exportFormats[item.MimeType])
assert.Equal(t, test.wantExtension, gotExtension) assert.Equal(t, test.wantExtension, gotExtension)
assert.Equal(t, test.wantMimeType, gotMimeType) assert.Equal(t, test.wantMimeType, gotMimeType)
} }