operations: ignore size of objects when they are < 0 #320

This allows google docs to be transferred and checked correctly.

docs/content/drive.md

@@ -382,13 +382,20 @@ see User rate limit exceeded errors, wait at least 24 hours and retry.
 You can disable server side copies with `--disable copy` to download
 and upload the files if you prefer.
 
+#### Limitations of Google Docs ####
+
 Google docs will appear as size -1 in `rclone ls` and as size 0 in
-anything which uses the VFS layer, eg `rclone mount`, `rclone serve
+anything which uses the VFS layer, eg `rclone mount`, `rclone serve`.
-XXX`.  This is because rclone can't find out the size of the Google
+
-Documents until they are downloaded.  An unfortunate consequence of
+This is because rclone can't find out the size of the Google docs
-this is that you can't download Google docs using `rclone mount` - you
+without downloading them.
-will get a 0 sized file.  If you try again the doc may gain its
+
-correct size and be downloadable.
+Google docs will transfer correctly with `rclone sync`, `rclone copy`
+etc as rclone knows to ignore the size when doing the transfer.
+
+However an unfortunate consequence of this is that you can't download
+Google docs using `rclone mount` - you will get a 0 sized file.  If
+you try again the doc may gain its correct size and be downloadable.
 
 ### Duplicated files ###
 
@@ -406,23 +413,9 @@ Android duplicates files on drive sometimes.
 
 ### Rclone appears to be re-copying files it shouldn't ###
 
-There are two possible reasons for rclone to recopy files which
+The most likely cause of this is the duplicated file issue above - run
-haven't changed to Google Drive.
+`rclone dedupe` and check your logs for duplicate object or directory
-
+messages.
-The first is the duplicated file issue above - run `rclone dedupe` and
-check your logs for duplicate object or directory messages.
-
-The second is that sometimes Google reports different sizes for the
-Google Docs exports which will cause rclone to re-download Google Docs
-for no apparent reason.  `--ignore-size` is a not very satisfactory
-work-around for this if it is causing you a lot of problems.
-
-### Google docs downloads sometimes fail with "Failed to copy: read X bytes expecting Y" ###
-
-This is the same problem as above.  Google reports the google doc is
-one size, but rclone downloads a different size.  Work-around with the
-`--ignore-size` flag or wait for rclone to retry the download which it
-will.
 
 ### Making your own client_id ###
 

fs/operations/operations.go

@@ -96,12 +96,19 @@ func Equal(src fs.ObjectInfo, dst fs.Object) bool {
 	return equal(src, dst, fs.Config.SizeOnly, fs.Config.CheckSum)
 }
 
+// sizeDiffers compare the size of src and dst taking into account the
+// various ways of ignoring sizes
+func sizeDiffers(src, dst fs.ObjectInfo) bool {
+	if fs.Config.IgnoreSize || src.Size() < 0 || dst.Size() < 0 {
+		return false
+	}
+	return src.Size() != dst.Size()
+}
+
 func equal(src fs.ObjectInfo, dst fs.Object, sizeOnly, checkSum bool) bool {
-	if !fs.Config.IgnoreSize {
+	if sizeDiffers(src, dst) {
-		if src.Size() != dst.Size() {
+		fs.Debugf(src, "Sizes differ (src %d vs dst %d)", src.Size(), dst.Size())
-			fs.Debugf(src, "Sizes differ (src %d vs dst %d)", src.Size(), dst.Size())
+		return false
-			return false
-		}
 	}
 	if sizeOnly {
 		fs.Debugf(src, "Sizes identical")
@@ -310,7 +317,7 @@ func Copy(f fs.Fs, dst fs.Object, remote string, src fs.Object) (newDst fs.Objec
 	}
 
 	// Verify sizes are the same after transfer
-	if !fs.Config.IgnoreSize && src.Size() != dst.Size() {
+	if sizeDiffers(src, dst) {
 		err = errors.Errorf("corrupted on transfer: sizes differ %d vs %d", src.Size(), dst.Size())
 		fs.Errorf(dst, "%v", err)
 		fs.CountError(err)
@@ -599,7 +606,7 @@ func (c *checkMarch) SrcOnly(src fs.DirEntry) (recurse bool) {
 func (c *checkMarch) checkIdentical(dst, src fs.Object) (differ bool, noHash bool) {
 	accounting.Stats.Checking(src.Remote())
 	defer accounting.Stats.DoneChecking(src.Remote())
-	if !fs.Config.IgnoreSize && src.Size() != dst.Size() {
+	if sizeDiffers(src, dst) {
 		err := errors.Errorf("Sizes differ")
 		fs.Errorf(src, "%v", err)
 		fs.CountError(err)

fs/operations/operations_internal_test.go

@@ -1,3 +1,40 @@
 // Internal tests for operations
 
 package operations
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/object"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSizeDiffers(t *testing.T) {
+	when := time.Now()
+	for _, test := range []struct {
+		ignoreSize bool
+		srcSize    int64
+		dstSize    int64
+		want       bool
+	}{
+		{false, 0, 0, false},
+		{false, 1, 2, true},
+		{false, 1, -1, false},
+		{false, -1, 1, false},
+		{true, 0, 0, false},
+		{true, 1, 2, false},
+		{true, 1, -1, false},
+		{true, -1, 1, false},
+	} {
+		src := object.NewStaticObjectInfo("a", when, test.srcSize, true, nil, nil)
+		dst := object.NewStaticObjectInfo("a", when, test.dstSize, true, nil, nil)
+		oldIgnoreSize := fs.Config.IgnoreSize
+		fs.Config.IgnoreSize = test.ignoreSize
+		got := sizeDiffers(src, dst)
+		fs.Config.IgnoreSize = oldIgnoreSize
+		assert.Equal(t, test.want, got, fmt.Sprintf("ignoreSize=%v, srcSize=%v, dstSize=%v", test.ignoreSize, test.srcSize, test.dstSize))
+	}
+}