drive: add --drive-chunk-size and --drive-upload-cutoff parameters

This commit is contained in:
Nick Craig-Wood 2015-03-14 17:55:38 +00:00
parent 5cc5429f99
commit 7c120b8bc5
2 changed files with 29 additions and 8 deletions

View File

@ -39,6 +39,10 @@ const (
var ( var (
// Flags // Flags
driveFullList = pflag.BoolP("drive-full-list", "", true, "Use a full listing for directory list. More data but usually quicker.") driveFullList = pflag.BoolP("drive-full-list", "", true, "Use a full listing for directory list. More data but usually quicker.")
// chunkSize is the size of the chunks created during a resumable upload and should be a power of two.
// 1<<18 is the minimum size supported by the Google uploader, and there is no maximum.
chunkSize = fs.SizeSuffix(256 * 1024)
driveUploadCutoff = chunkSize
// Description of how to auth for this app // Description of how to auth for this app
driveAuth = &googleauth.Auth{ driveAuth = &googleauth.Auth{
Scope: "https://www.googleapis.com/auth/drive", Scope: "https://www.googleapis.com/auth/drive",
@ -63,6 +67,8 @@ func init() {
Help: "Google Application Client Secret - leave blank to use rclone's.", Help: "Google Application Client Secret - leave blank to use rclone's.",
}}, }},
}) })
pflag.VarP(&driveUploadCutoff, "drive-upload-cutoff", "", "Cutoff for switching to chunked upload")
pflag.VarP(&chunkSize, "drive-chunk-size", "", "Upload chunk size. Must a power of 2 >= 256k.")
} }
// FsDrive represents a remote drive server // FsDrive represents a remote drive server
@ -293,8 +299,27 @@ OUTER:
return return
} }
// Returns true of x is a power of 2 or zero
func isPowerOfTwo(x int64) bool {
switch {
case x == 0:
return true
case x < 0:
return false
default:
return (x & (x - 1)) == 0
}
}
// NewFs contstructs an FsDrive from the path, container:path // NewFs contstructs an FsDrive from the path, container:path
func NewFs(name, path string) (fs.Fs, error) { func NewFs(name, path string) (fs.Fs, error) {
if !isPowerOfTwo(int64(chunkSize)) {
return nil, fmt.Errorf("drive: chunk size %v isn't a power of two", chunkSize)
}
if chunkSize < 256*1024 {
return nil, fmt.Errorf("drive: chunk size can't be less than 256k - was %v", chunkSize)
}
t, err := driveAuth.NewTransport(name) t, err := driveAuth.NewTransport(name)
if err != nil { if err != nil {
return nil, err return nil, err
@ -730,7 +755,7 @@ func (f *FsDrive) Put(in io.Reader, remote string, modTime time.Time, size int64
} }
var info *drive.File var info *drive.File
if false { if size == 0 || size < int64(driveUploadCutoff) {
// Make the API request to upload metadata and file data. // Make the API request to upload metadata and file data.
// Don't retry, return a retry error instead // Don't retry, return a retry error instead
f.beginCall() f.beginCall()
@ -970,7 +995,7 @@ func (o *FsObjectDrive) Update(in io.Reader, modTime time.Time, size int64) erro
// Make the API request to upload metadata and file data. // Make the API request to upload metadata and file data.
var err error var err error
var info *drive.File var info *drive.File
if false { if size == 0 || size < int64(driveUploadCutoff) {
// Don't retry, return a retry error instead // Don't retry, return a retry error instead
o.drive.beginCall() o.drive.beginCall()
info, err = o.drive.svc.Files.Update(updateInfo.Id, updateInfo).SetModifiedDate(true).Media(in).Do() info, err = o.drive.svc.Files.Update(updateInfo.Id, updateInfo).SetModifiedDate(true).Media(in).Do()

View File

@ -31,10 +31,6 @@ const (
// Number of times to try each chunk // Number of times to try each chunk
maxTries = 10 maxTries = 10
// chunkSize is the size of the chunks created during a resumable upload and should be a power of two.
// 1<<18 is the minimum size supported by the Google uploader, and there is no maximum.
chunkSize int64 = 1 << 18
) )
// resumableUpload is used by the generated APIs to provide resumable uploads. // resumableUpload is used by the generated APIs to provide resumable uploads.
@ -194,8 +190,8 @@ func (rx *resumableUpload) Upload() (*drive.File, error) {
var StatusCode int var StatusCode int
for start < rx.ContentLength { for start < rx.ContentLength {
reqSize := rx.ContentLength - start reqSize := rx.ContentLength - start
if reqSize >= chunkSize { if reqSize >= int64(chunkSize) {
reqSize = chunkSize reqSize = int64(chunkSize)
} else { } else {
buf = buf[:reqSize] buf = buf[:reqSize]
} }