lib/rest: Calculate correct Content-Length on MultiPart Uploads

This is used in the pcloud backend and in the upcoming 1fichier backend.
This commit is contained in:
Fionera 2019-06-24 20:34:49 +02:00 committed by Nick Craig-Wood
parent 07e2c3a50f
commit 6cd7c3b774
2 changed files with 42 additions and 15 deletions

View File

@ -9,11 +9,9 @@ package pcloud
// FIXME mime type? Fix overview if implement.
import (
"bytes"
"context"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"net/url"
@ -1091,21 +1089,18 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
// opts.Body=0), so upload it as a multpart form POST with
// Content-Length set.
if size == 0 {
formReader, contentType, err := rest.MultipartUpload(in, opts.Parameters, "content", leaf)
formReader, contentType, overhead, err := rest.MultipartUpload(in, opts.Parameters, "content", leaf)
if err != nil {
return errors.Wrap(err, "failed to make multipart upload for 0 length file")
}
formBody, err := ioutil.ReadAll(formReader)
if err != nil {
return errors.Wrap(err, "failed to read multipart upload for 0 length file")
}
length := int64(len(formBody))
contentLength := overhead + size
opts.ContentType = contentType
opts.Body = bytes.NewBuffer(formBody)
opts.Body = formReader
opts.Method = "POST"
opts.Parameters = nil
opts.ContentLength = &length
opts.ContentLength = &contentLength
}
err = o.fs.pacer.CallNoRetry(func() (bool, error) {

View File

@ -291,12 +291,42 @@ func (api *Client) Call(opts *Opts) (resp *http.Response, err error) {
// fileName - is the name of the attached file
// contentName - the name of the parameter for the file
//
// the *int64 returned is the overhead in addition to the file contents, in case Content-Length is required
//
// NB This doesn't allow setting the content type of the attachment
func MultipartUpload(in io.Reader, params url.Values, contentName, fileName string) (io.ReadCloser, string, error) {
func MultipartUpload(in io.Reader, params url.Values, contentName, fileName string) (io.ReadCloser, string, int64, error) {
bodyReader, bodyWriter := io.Pipe()
writer := multipart.NewWriter(bodyWriter)
contentType := writer.FormDataContentType()
// Create a Multipart Writer as base for calculating the Content-Length
buf := &bytes.Buffer{}
dummyMultipartWriter := multipart.NewWriter(buf)
err := dummyMultipartWriter.SetBoundary(writer.Boundary())
if err != nil {
return nil, "", 0, err
}
for key, vals := range params {
for _, val := range vals {
err := dummyMultipartWriter.WriteField(key, val)
if err != nil {
return nil, "", 0, err
}
}
}
_, err = dummyMultipartWriter.CreateFormFile(contentName, fileName)
if err != nil {
return nil, "", 0, err
}
err = dummyMultipartWriter.Close()
if err != nil {
return nil, "", 0, err
}
multipartLength := int64(buf.Len())
// Pump the data in the background
go func() {
var err error
@ -332,7 +362,7 @@ func MultipartUpload(in io.Reader, params url.Values, contentName, fileName stri
_ = bodyWriter.Close()
}()
return bodyReader, contentType, nil
return bodyReader, contentType, multipartLength, nil
}
// CallJSON runs Call and decodes the body as a JSON object into response (if not nil)
@ -403,9 +433,11 @@ func (api *Client) callCodec(opts *Opts, request interface{}, response interface
params.Add(opts.MultipartMetadataName, string(requestBody))
}
opts = opts.Copy()
opts.Body, opts.ContentType, err = MultipartUpload(opts.Body, params, opts.MultipartContentName, opts.MultipartFileName)
if err != nil {
return nil, err
var overhead int64
opts.Body, opts.ContentType, overhead, err = MultipartUpload(opts.Body, params, opts.MultipartContentName, opts.MultipartFileName)
if opts.ContentLength != nil {
*opts.ContentLength += overhead
}
}
resp, err = api.Call(opts)