community
/
rclone
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

drive: request the export formats only when required #320 

If the listing has no google docs in or the user uses
`--drive-skip-gdocs` then we don't fetch the export formats which
saves a transaction to drive.
This commit is contained in:
Nick Craig-Wood 2018-01-31 20:03:02 +00:00
parent 3c7a755631
commit a6227f34e2
2 changed files with 107 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

100
backend/drive/drive.go
View File

@ -19,6 +19,7 @@ import (
"sort"
"strconv"
"strings"
"sync"
"time"
"github.com/ncw/rclone/fs"
@ -97,6 +98,8 @@ var (
}
extensionToMimeType map[string]string
partialFields = "id,name,size,md5Checksum,trashed,modifiedTime,mimeType"
exportFormatsOnce sync.Once // make sure we fetch the export formats only once
_exportFormats map[string][]string // allowed export mime-type conversions
)
// Register with Fs
@ -176,18 +179,17 @@ func init() {
// Fs represents a remote drive server
type Fs struct {
name string // name of this remote
root string // the path we are working on
features *fs.Features // optional features
svc *drive.Service // the connection to the drive server
client *http.Client // authorized client
rootFolderID string // the id of the root folder
dirCache *dircache.DirCache // Map of directory path to directory id
pacer *pacer.Pacer // To pace the API calls
extensions []string // preferred extensions to download docs
exportFormats map[string][]string // allowed export mime-type conversions
teamDriveID string // team drive ID, may be ""
isTeamDrive bool // true if this is a team drive
name string // name of this remote
root string // the path we are working on
features *fs.Features // optional features
svc *drive.Service // the connection to the drive server
client *http.Client // authorized client
rootFolderID string // the id of the root folder
dirCache *dircache.DirCache // Map of directory path to directory id
pacer *pacer.Pacer // To pace the API calls
extensions []string // preferred extensions to download docs
teamDriveID string // team drive ID, may be ""
isTeamDrive bool // true if this is a team drive
}
// Object describes a drive object
@ -530,16 +532,6 @@ func NewFs(name, path string) (fs.Fs, error) {
f.dirCache = dircache.New(root, f.rootFolderID, f)
var about *drive.About
err = f.pacer.Call(func() (bool, error) {
about, err = f.svc.About.Get().Fields("exportFormats").Do()
return shouldRetry(err)
})
if err != nil {
return nil, errors.Wrap(err, "couldn't get Drive exportFormats")
}
f.exportFormats = about.ExportFormats
// Parse extensions
err = f.parseExtensions(*driveExtensions)
if err != nil {
@ -651,6 +643,28 @@ func isAuthOwned(item *drive.File) bool {
return false
}
// exportFormats returns the export formats from drive, fetching them
// if necessary.
//
// if the fetch fails then it will not export any drive formats
func (f *Fs) exportFormats() map[string][]string {
exportFormatsOnce.Do(func() {
var about *drive.About
var err error
err = f.pacer.Call(func() (bool, error) {
about, err = f.svc.About.Get().Fields("exportFormats").Do()
return shouldRetry(err)
})
if err != nil {
fs.Errorf(f, "Failed to get Drive exportFormats: %v", err)
_exportFormats = map[string][]string{}
return
}
_exportFormats = about.ExportFormats
})
return _exportFormats
}
// findExportFormat works out the optimum extension and mime-type
// for this item.
//
@ -692,7 +706,6 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
var iErr error
_, err = f.list(directoryID, "", false, false, false, func(item *drive.File) bool {
remote := path.Join(dir, item.Name)
exportMimeTypes, isDocument := f.exportFormats[item.MimeType]
switch {
case *driveAuthOwnerOnly && !isAuthOwned(item):
// ignore object or directory
@ -710,30 +723,31 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
return true
}
entries = append(entries, o)
case isDocument:
case *driveSkipGdocs:
fs.Debugf(remote, "Skipping google document type %q", item.MimeType)
default:
exportMimeTypes, isDocument := f.exportFormats()[item.MimeType]
if !isDocument {
fs.Debugf(remote, "Ignoring unknown document type %q", item.MimeType)
break
}
// If item has export links then it is a google doc
extension, exportMimeType := f.findExportFormat(remote, exportMimeTypes)
if extension == "" {
fs.Debugf(remote, "No export formats found")
} else {
o, err := f.newObjectWithInfo(remote+"."+extension, item)
if err != nil {
iErr = err
return true
}
if !*driveSkipGdocs {
obj := o.(*Object)
obj.url = fmt.Sprintf("%sfiles/%s/export?mimeType=%s", f.svc.BasePath, item.Id, url.QueryEscape(exportMimeType))
obj.isDocument = true
obj.mimeType = exportMimeType
obj.bytes = -1
entries = append(entries, o)
} else {
fs.Debugf(f, "Skip google document: %q", remote)
}
fs.Debugf(remote, "No export formats found for %q", item.MimeType)
break
}
default:
fs.Debugf(remote, "Ignoring unknown object")
o, err := f.newObjectWithInfo(remote+"."+extension, item)
if err != nil {
iErr = err
return true
}
obj := o.(*Object)
obj.url = fmt.Sprintf("%sfiles/%s/export?mimeType=%s", f.svc.BasePath, item.Id, url.QueryEscape(exportMimeType))
obj.isDocument = true
obj.mimeType = exportMimeType
obj.bytes = -1
entries = append(entries, o)
}
return false
})

94
backend/drive/drive_internal_test.go
View File

@ -10,48 +10,55 @@ import (
"github.com/stretchr/testify/assert"
)
const exportFormats = `{
"application/vnd.google-apps.document": [
"application/rtf",
"application/vnd.oasis.opendocument.text",
"text/html",
"application/pdf",
"application/epub+zip",
"application/zip",
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
"text/plain"
],
"application/vnd.google-apps.spreadsheet": [
"application/x-vnd.oasis.opendocument.spreadsheet",
"text/tab-separated-values",
"application/pdf",
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"text/csv",
"application/zip",
"application/vnd.oasis.opendocument.spreadsheet"
],
"application/vnd.google-apps.jam": [
"application/pdf"
],
"application/vnd.google-apps.script": [
"application/vnd.google-apps.script+json"
],
"application/vnd.google-apps.presentation": [
"application/vnd.oasis.opendocument.presentation",
"application/pdf",
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
"text/plain"
],
"application/vnd.google-apps.form": [
"application/zip"
],
"application/vnd.google-apps.drawing": [
"image/svg+xml",
"image/png",
"application/pdf",
"image/jpeg"
]
}`
const exampleExportFormats = `{
"application/vnd.google-apps.document": [
"application/rtf",
"application/vnd.oasis.opendocument.text",
"text/html",
"application/pdf",
"application/epub+zip",
"application/zip",
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
"text/plain"
],
"application/vnd.google-apps.spreadsheet": [
"application/x-vnd.oasis.opendocument.spreadsheet",
"text/tab-separated-values",
"application/pdf",
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"text/csv",
"application/zip",
"application/vnd.oasis.opendocument.spreadsheet"
],
"application/vnd.google-apps.jam": [
"application/pdf"
],
"application/vnd.google-apps.script": [
"application/vnd.google-apps.script+json"
],
"application/vnd.google-apps.presentation": [
"application/vnd.oasis.opendocument.presentation",
"application/pdf",
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
"text/plain"
],
"application/vnd.google-apps.form": [
"application/zip"
],
"application/vnd.google-apps.drawing": [
"image/svg+xml",
"image/png",
"application/pdf",
"image/jpeg"
]
}`
var exportFormats map[string][]string
// Load the example export formats into exportFormats for testing
func TestInternalLoadExampleExportFormats(t *testing.T) {
assert.NoError(t, json.Unmarshal([]byte(exampleExportFormats), &exportFormats))
}
func TestInternalParseExtensions(t *testing.T) {
for _, test := range []struct {
@ -98,8 +105,7 @@ func TestInternalFindExportFormat(t *testing.T) {
} {
f := new(Fs)
f.extensions = test.extensions
assert.NoError(t, json.Unmarshal([]byte(exportFormats), &f.exportFormats))
gotExtension, gotMimeType := f.findExportFormat("file", f.exportFormats[item.MimeType])
gotExtension, gotMimeType := f.findExportFormat("file", exportFormats[item.MimeType])
assert.Equal(t, test.wantExtension, gotExtension)
assert.Equal(t, test.wantMimeType, gotMimeType)
}