community
/
rclone
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

drive: code cleanup

This commit is contained in:
Fabian Möller 2018-09-01 13:16:01 +02:00 committed by Nick Craig-Wood
parent 15b1a1f909
commit a20fae0364
2 changed files with 48 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

75
backend/drive/drive.go
View File

@ -58,6 +58,7 @@ const (
// chunkSize is the size of the chunks created during a resumable upload and should be a power of two. // chunkSize is the size of the chunks created during a resumable upload and should be a power of two.
// 1<<18 is the minimum size supported by the Google uploader, and there is no maximum. // 1<<18 is the minimum size supported by the Google uploader, and there is no maximum.
defaultChunkSize = fs.SizeSuffix(8 * 1024 * 1024) defaultChunkSize = fs.SizeSuffix(8 * 1024 * 1024)
partialFields = "id,name,size,md5Checksum,trashed,modifiedTime,createdTime,mimeType,parents,webViewLink"
) )
// Globals // Globals
@ -113,7 +114,6 @@ var (
_mimeTypeCustomTransform = map[string]string{ _mimeTypeCustomTransform = map[string]string{
"application/vnd.google-apps.script+json": "application/json", "application/vnd.google-apps.script+json": "application/json",
} }
partialFields = "id,name,size,md5Checksum,trashed,modifiedTime,createdTime,mimeType,parents,webViewLink"
fetchFormatsOnce sync.Once // make sure we fetch the export/import formats only once fetchFormatsOnce sync.Once // make sure we fetch the export/import formats only once
_exportFormats map[string][]string // allowed export MIME type conversions _exportFormats map[string][]string // allowed export MIME type conversions
_importFormats map[string][]string // allowed import MIME type conversions _importFormats map[string][]string // allowed import MIME type conversions
@ -402,27 +402,27 @@ func (f *Fs) Features() *fs.Features {
} }
// shouldRetry determines whehter a given err rates being retried // shouldRetry determines whehter a given err rates being retried
func shouldRetry(err error) (again bool, errOut error) { func shouldRetry(err error) (bool, error) {
again = false if err == nil {
if err != nil { return false, nil
}
if fserrors.ShouldRetry(err) { if fserrors.ShouldRetry(err) {
again = true return true, err
} else { }
switch gerr := err.(type) { switch gerr := err.(type) {
case *googleapi.Error: case *googleapi.Error:
if gerr.Code >= 500 && gerr.Code < 600 { if gerr.Code >= 500 && gerr.Code < 600 {
// All 5xx errors should be retried // All 5xx errors should be retried
again = true return true, err
} else if len(gerr.Errors) > 0 { }
if len(gerr.Errors) > 0 {
reason := gerr.Errors[0].Reason reason := gerr.Errors[0].Reason
if reason == "rateLimitExceeded" || reason == "userRateLimitExceeded" { if reason == "rateLimitExceeded" || reason == "userRateLimitExceeded" {
again = true return true, err
} }
} }
} }
} return false, err
}
return again, err
} }
// parseParse parses a drive 'url' // parseParse parses a drive 'url'
@ -450,7 +450,7 @@ func containsString(slice []string, s string) bool {
// If the user fn ever returns true then it early exits with found = true // If the user fn ever returns true then it early exits with found = true
// //
// Search params: https://developers.google.com/drive/search-parameters // Search params: https://developers.google.com/drive/search-parameters
func (f *Fs) list(dirIDs []string, title string, directoriesOnly bool, filesOnly bool, includeAll bool, fn listFn) (found bool, err error) { func (f *Fs) list(dirIDs []string, title string, directoriesOnly, filesOnly, includeAll bool, fn listFn) (found bool, err error) {
var query []string var query []string
if !includeAll { if !includeAll {
q := "trashed=" + strconv.FormatBool(f.opt.TrashedOnly) q := "trashed=" + strconv.FormatBool(f.opt.TrashedOnly)
@ -641,14 +641,7 @@ func parseExtensions(extensionsIn ...string) (extensions, mimeTypes []string, er
if mt == "" { if mt == "" {
return extensions, mimeTypes, errors.Errorf("couldn't find MIME type for extension %q", extension) return extensions, mimeTypes, errors.Errorf("couldn't find MIME type for extension %q", extension)
} }
found := false if !containsString(extensions, extension) {
for _, existingExtension := range extensions {
if extension == existingExtension {
found = true
break
}
}
if !found {
extensions = append(extensions, extension) extensions = append(extensions, extension)
mimeTypes = append(mimeTypes, mt) mimeTypes = append(mimeTypes, mt)
} }
@ -1045,7 +1038,7 @@ func (f *Fs) CreateDir(pathID, leaf string) (newID string, err error) {
var info *drive.File var info *drive.File
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
info, err = f.svc.Files.Create(createInfo). info, err = f.svc.Files.Create(createInfo).
Fields(googleapi.Field(partialFields)). Fields("id").
SupportsTeamDrives(f.isTeamDrive). SupportsTeamDrives(f.isTeamDrive).
Do() Do()
return shouldRetry(err) return shouldRetry(err)
@ -1209,6 +1202,7 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
_, err = f.list([]string{directoryID}, "", false, false, false, func(item *drive.File) bool { _, err = f.list([]string{directoryID}, "", false, false, false, func(item *drive.File) bool {
entry, err := f.itemToDirEntry(path.Join(dir, item.Name), item) entry, err := f.itemToDirEntry(path.Join(dir, item.Name), item)
if err != nil { if err != nil {
iErr = err
return true return true
} }
if entry != nil { if entry != nil {
@ -1502,7 +1496,7 @@ func (f *Fs) PutUnchecked(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOpt
err = f.pacer.CallNoRetry(func() (bool, error) { err = f.pacer.CallNoRetry(func() (bool, error) {
info, err = f.svc.Files.Create(createInfo). info, err = f.svc.Files.Create(createInfo).
Media(in, googleapi.ContentType(srcMimeType)). Media(in, googleapi.ContentType(srcMimeType)).
Fields(googleapi.Field(partialFields)). Fields(partialFields).
SupportsTeamDrives(f.isTeamDrive). SupportsTeamDrives(f.isTeamDrive).
KeepRevisionForever(f.opt.KeepRevisionForever). KeepRevisionForever(f.opt.KeepRevisionForever).
Do() Do()
@ -1513,7 +1507,7 @@ func (f *Fs) PutUnchecked(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOpt
} }
} else { } else {
// Upload the file in chunks // Upload the file in chunks
info, err = f.Upload(in, size, srcMimeType, "", createInfo, remote) info, err = f.Upload(in, size, srcMimeType, "", remote, createInfo)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -1657,17 +1651,14 @@ func (f *Fs) Precision() time.Duration {
// If it isn't possible then return fs.ErrorCantCopy // If it isn't possible then return fs.ErrorCantCopy
func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) { func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
var srcObj *baseObject var srcObj *baseObject
srcRemote := src.Remote()
ext := "" ext := ""
switch src := src.(type) { switch src := src.(type) {
case *Object: case *Object:
srcObj = &src.baseObject srcObj = &src.baseObject
case *documentObject: case *documentObject:
srcObj = &src.baseObject srcObj, ext = &src.baseObject, src.ext()
srcRemote, ext = srcRemote[:len(srcRemote)-src.extLen], srcRemote[len(srcRemote)-src.extLen:]
case *linkObject: case *linkObject:
srcObj = &src.baseObject srcObj, ext = &src.baseObject, src.ext()
srcRemote, ext = srcRemote[:len(srcRemote)-src.extLen], srcRemote[len(srcRemote)-src.extLen:]
default: default:
fs.Debugf(src, "Can't copy - not same remote type") fs.Debugf(src, "Can't copy - not same remote type")
return nil, fs.ErrorCantCopy return nil, fs.ErrorCantCopy
@ -1689,7 +1680,7 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
var info *drive.File var info *drive.File
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
info, err = f.svc.Files.Copy(srcObj.id, createInfo). info, err = f.svc.Files.Copy(srcObj.id, createInfo).
Fields(googleapi.Field(partialFields)). Fields(partialFields).
SupportsTeamDrives(f.isTeamDrive). SupportsTeamDrives(f.isTeamDrive).
KeepRevisionForever(f.opt.KeepRevisionForever). KeepRevisionForever(f.opt.KeepRevisionForever).
Do() Do()
@ -1790,17 +1781,14 @@ func (f *Fs) About() (*fs.Usage, error) {
// If it isn't possible then return fs.ErrorCantMove // If it isn't possible then return fs.ErrorCantMove
func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) { func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
var srcObj *baseObject var srcObj *baseObject
srcRemote := src.Remote()
ext := "" ext := ""
switch src := src.(type) { switch src := src.(type) {
case *Object: case *Object:
srcObj = &src.baseObject srcObj = &src.baseObject
case *documentObject: case *documentObject:
srcObj = &src.baseObject srcObj, ext = &src.baseObject, src.ext()
srcRemote, ext = srcRemote[:len(srcRemote)-src.extLen], srcRemote[len(srcRemote)-src.extLen:]
case *linkObject: case *linkObject:
srcObj = &src.baseObject srcObj, ext = &src.baseObject, src.ext()
srcRemote, ext = srcRemote[:len(srcRemote)-src.extLen], srcRemote[len(srcRemote)-src.extLen:]
default: default:
fs.Debugf(src, "Can't move - not same remote type") fs.Debugf(src, "Can't move - not same remote type")
return nil, fs.ErrorCantMove return nil, fs.ErrorCantMove
@ -1833,7 +1821,7 @@ func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
info, err = f.svc.Files.Update(srcObj.id, dstInfo). info, err = f.svc.Files.Update(srcObj.id, dstInfo).
RemoveParents(srcParentID). RemoveParents(srcParentID).
AddParents(dstParents). AddParents(dstParents).
Fields(googleapi.Field(partialFields)). Fields(partialFields).
SupportsTeamDrives(f.isTeamDrive). SupportsTeamDrives(f.isTeamDrive).
Do() Do()
return shouldRetry(err) return shouldRetry(err)
@ -1869,7 +1857,7 @@ func (f *Fs) PublicLink(remote string) (link string, err error) {
// TODO: On TeamDrives this might fail if lacking permissions to change ACLs. // TODO: On TeamDrives this might fail if lacking permissions to change ACLs.
// Need to either check `canShare` attribute on the object or see if a sufficient permission is already present. // Need to either check `canShare` attribute on the object or see if a sufficient permission is already present.
_, err = f.svc.Permissions.Create(id, permission). _, err = f.svc.Permissions.Create(id, permission).
Fields(googleapi.Field("id")). Fields("").
SupportsTeamDrives(f.isTeamDrive). SupportsTeamDrives(f.isTeamDrive).
Do() Do()
return shouldRetry(err) return shouldRetry(err)
@ -2249,7 +2237,7 @@ func (o *baseObject) SetModTime(modTime time.Time) error {
err := o.fs.pacer.Call(func() (bool, error) { err := o.fs.pacer.Call(func() (bool, error) {
var err error var err error
info, err = o.fs.svc.Files.Update(o.id, updateInfo). info, err = o.fs.svc.Files.Update(o.id, updateInfo).
Fields(googleapi.Field(partialFields)). Fields(partialFields).
SupportsTeamDrives(o.fs.isTeamDrive). SupportsTeamDrives(o.fs.isTeamDrive).
Do() Do()
return shouldRetry(err) return shouldRetry(err)
@ -2437,7 +2425,7 @@ func (o *baseObject) update(updateInfo *drive.File, uploadMimeType string, in io
err = o.fs.pacer.CallNoRetry(func() (bool, error) { err = o.fs.pacer.CallNoRetry(func() (bool, error) {
info, err = o.fs.svc.Files.Update(o.id, updateInfo). info, err = o.fs.svc.Files.Update(o.id, updateInfo).
Media(in, googleapi.ContentType(uploadMimeType)). Media(in, googleapi.ContentType(uploadMimeType)).
Fields(googleapi.Field(partialFields)). Fields(partialFields).
SupportsTeamDrives(o.fs.isTeamDrive). SupportsTeamDrives(o.fs.isTeamDrive).
KeepRevisionForever(o.fs.opt.KeepRevisionForever). KeepRevisionForever(o.fs.opt.KeepRevisionForever).
Do() Do()
@ -2446,7 +2434,7 @@ func (o *baseObject) update(updateInfo *drive.File, uploadMimeType string, in io
return return
} }
// Upload the file in chunks // Upload the file in chunks
return o.fs.Upload(in, size, uploadMimeType, o.id, updateInfo, o.remote) return o.fs.Upload(in, size, uploadMimeType, o.id, o.remote, updateInfo)
} }
// Update the already existing object // Update the already existing object
@ -2550,6 +2538,13 @@ func (o *baseObject) ID() string {
return o.id return o.id
} }
func (o *documentObject) ext() string {
return o.baseObject.remote[len(o.baseObject.remote)-o.extLen:]
}
func (o *linkObject) ext() string {
return o.baseObject.remote[len(o.baseObject.remote)-o.extLen:]
}
// templates for document link files // templates for document link files
const ( const (
urlTemplate = `[InternetShortcut]{{"\r"}} urlTemplate = `[InternetShortcut]{{"\r"}}

11
backend/drive/upload.go
View File

@ -50,11 +50,12 @@ type resumableUpload struct {
} }
// Upload the io.Reader in of size bytes with contentType and info // Upload the io.Reader in of size bytes with contentType and info
func (f *Fs) Upload(in io.Reader, size int64, contentType string, fileID string, info *drive.File, remote string) (*drive.File, error) { func (f *Fs) Upload(in io.Reader, size int64, contentType, fileID, remote string, info *drive.File) (*drive.File, error) {
params := make(url.Values) params := url.Values{
params.Set("alt", "json") "alt": {"json"},
params.Set("uploadType", "resumable") "uploadType": {"resumable"},
params.Set("fields", partialFields) "fields": {partialFields},
}
if f.isTeamDrive { if f.isTeamDrive {
params.Set("supportsTeamDrives", "true") params.Set("supportsTeamDrives", "true")
} }