copyurl: honor --no-check-certificate

fs/fshttp/http.go

@@ -28,7 +28,7 @@ const (
 
 var (
 	transport    http.RoundTripper
-	noTransport  sync.Once
+	noTransport  = new(sync.Once)
 	tpsBucket    *rate.Limiter // for limiting number of http transactions per second
 	cookieJar, _ = cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List})
 )
@@ -121,9 +121,15 @@ func dialContextTimeout(ctx context.Context, network, address string, ci *fs.Con
 	return newTimeoutConn(c, ci.Timeout)
 }
 
+// ResetTransport resets the existing transport, allowing it to take new settings.
+// Should only be used for testing.
+func ResetTransport() {
+	noTransport = new(sync.Once)
+}
+
 // NewTransport returns an http.RoundTripper with the correct timeouts
 func NewTransport(ci *fs.ConfigInfo) http.RoundTripper {
-	noTransport.Do(func() {
+	(*noTransport).Do(func() {
 		// Start with a sensible set of defaults then override.
 		// This also means we get new stuff when it gets added to go
 		t := new(http.Transport)

fs/operations/operations.go

@@ -8,7 +8,6 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"net/http"
 	"path"
 	"path/filepath"
 	"sort"
@@ -21,6 +20,7 @@ import (
 	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fs/accounting"
 	"github.com/ncw/rclone/fs/fserrors"
+	"github.com/ncw/rclone/fs/fshttp"
 	"github.com/ncw/rclone/fs/hash"
 	"github.com/ncw/rclone/fs/march"
 	"github.com/ncw/rclone/fs/object"
@@ -1411,7 +1411,9 @@ func RcatSize(fdst fs.Fs, dstFileName string, in io.ReadCloser, size int64, modT
 
 // CopyURL copies the data from the url to (fdst, dstFileName)
 func CopyURL(fdst fs.Fs, dstFileName string, url string) (dst fs.Object, err error) {
-	resp, err := http.Get(url)
+	client := fshttp.NewClient(fs.Config)
+	resp, err := client.Get(url)
+
 	if err != nil {
 		return nil, err
 	}

fs/operations/operations_test.go

@@ -38,6 +38,7 @@ import (
 	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fs/accounting"
 	"github.com/ncw/rclone/fs/filter"
+	"github.com/ncw/rclone/fs/fshttp"
 	"github.com/ncw/rclone/fs/hash"
 	"github.com/ncw/rclone/fs/operations"
 	"github.com/ncw/rclone/fstest"
@@ -673,15 +674,18 @@ func TestCopyURL(t *testing.T) {
 	r := fstest.NewRun(t)
 	defer r.Finalise()
 
-	contents := "file1 contents\n"
+	contents := "file contents\n"
 	file1 := r.WriteFile("file1", contents, t1)
+	file2 := r.WriteFile("file2", contents, t1)
 	r.Mkdir(r.Fremote)
 	fstest.CheckItems(t, r.Fremote)
 
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	// check when reading from regular HTTP server
+	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		_, err := w.Write([]byte(contents))
 		assert.NoError(t, err)
-	}))
+	})
+	ts := httptest.NewServer(handler)
 	defer ts.Close()
 
 	o, err := operations.CopyURL(r.Fremote, "file1", ts.URL)
@@ -689,6 +693,21 @@ func TestCopyURL(t *testing.T) {
 	assert.Equal(t, int64(len(contents)), o.Size())
 
 	fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1}, nil, fs.ModTimeNotSupported)
+
+	// check when reading from unverified HTTPS server
+	fs.Config.InsecureSkipVerify = true
+	fshttp.ResetTransport()
+	defer func() {
+		fs.Config.InsecureSkipVerify = false
+		fshttp.ResetTransport()
+	}()
+	tss := httptest.NewTLSServer(handler)
+	defer tss.Close()
+
+	o, err = operations.CopyURL(r.Fremote, "file2", tss.URL)
+	require.NoError(t, err)
+	assert.Equal(t, int64(len(contents)), o.Size())
+	fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1, file2}, nil, fs.ModTimeNotSupported)
 }
 
 func TestMoveFile(t *testing.T) {