From 661dc568f37a4a01195ded16969d3a6ba30a7f7d Mon Sep 17 00:00:00 2001 From: Ivan Andreev Date: Mon, 16 Sep 2019 20:52:41 +0300 Subject: [PATCH] fstest: let backends advertise maximum file size --- fs/operations/multithread_test.go | 3 +++ fs/operations/operations_test.go | 9 +++++++++ fstest/fstest.go | 2 ++ fstest/fstests/fstests.go | 9 +++++++-- fstest/test_all/config.go | 8 ++++++++ fstest/test_all/run.go | 5 +++++ 6 files changed, 34 insertions(+), 2 deletions(-) diff --git a/fs/operations/multithread_test.go b/fs/operations/multithread_test.go index a54b19f5e..9e125a702 100644 --- a/fs/operations/multithread_test.go +++ b/fs/operations/multithread_test.go @@ -115,6 +115,9 @@ func TestMultithreadCopy(t *testing.T) { {size: multithreadChunkSize*2 + 1, streams: 2}, } { t.Run(fmt.Sprintf("%+v", test), func(t *testing.T) { + if *fstest.SizeLimit > 0 && int64(test.size) > *fstest.SizeLimit { + t.Skipf("exceeded file size limit %d > %d", test.size, *fstest.SizeLimit) + } var err error contents := random.String(test.size) t1 := fstest.Time("2001-02-03T04:05:06.499999999Z") diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go index 526690820..8fded2ebd 100644 --- a/fs/operations/operations_test.go +++ b/fs/operations/operations_test.go @@ -445,6 +445,15 @@ func TestRcat(t *testing.T) { r := fstest.NewRun(t) defer r.Finalise() + if *fstest.SizeLimit > 0 && int64(fs.Config.StreamingUploadCutoff) > *fstest.SizeLimit { + savedCutoff := fs.Config.StreamingUploadCutoff + defer func() { + fs.Config.StreamingUploadCutoff = savedCutoff + }() + fs.Config.StreamingUploadCutoff = fs.SizeSuffix(*fstest.SizeLimit) + t.Logf("Adjust StreamingUploadCutoff to size limit %s (was %s)", fs.Config.StreamingUploadCutoff, savedCutoff) + } + fstest.CheckListing(t, r.Fremote, []fstest.Item{}) data1 := "this is some really nice test data" diff --git a/fstest/fstest.go b/fstest/fstest.go index 1924379c7..509d034bb 100644 --- a/fstest/fstest.go +++ b/fstest/fstest.go @@ -42,6 +42,8 @@ var ( Individual = flag.Bool("individual", false, "Make individual bucket/container/directory for each test - much slower") LowLevelRetries = flag.Int("low-level-retries", 10, "Number of low level retries") UseListR = flag.Bool("fast-list", false, "Use recursive list if available. Uses more memory but fewer transactions.") + // SizeLimit signals tests to skip maximum test file size and skip inappropriate runs + SizeLimit = flag.Int64("size-limit", 0, "Limit maximum test file size") // ListRetries is the number of times to retry a listing to overcome eventual consistency ListRetries = flag.Int("list-retries", 6, "Number or times to retry listing") // MatchTestRemote matches the remote names used for testing diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go index 80983a777..2505473d6 100644 --- a/fstest/fstests/fstests.go +++ b/fstest/fstests/fstests.go @@ -557,9 +557,14 @@ func Run(t *testing.T, opt *Opt) { t.Run("FsPutError", func(t *testing.T) { skipIfNotOk(t) - const N = 5 * 1024 + var N int64 = 5 * 1024 + if *fstest.SizeLimit > 0 && N > *fstest.SizeLimit { + N = *fstest.SizeLimit + t.Logf("Reduce file size due to limit %d", N) + } + // Read N bytes then produce an error - contents := random.String(N) + contents := random.String(int(N)) buf := bytes.NewBufferString(contents) er := &errorReader{errors.New("potato")} in := io.MultiReader(buf, er) diff --git a/fstest/test_all/config.go b/fstest/test_all/config.go index 2fe0b7a6f..2f365987d 100644 --- a/fstest/test_all/config.go +++ b/fstest/test_all/config.go @@ -34,6 +34,7 @@ type Backend struct { FastList bool // set to test with -fast-list Short bool // set to test with -short OneOnly bool // set to run only one backend test at once + MaxFile string // file size limit Ignore []string // test names to ignore the failure of Tests []string // paths of tests to run, blank for all } @@ -60,6 +61,12 @@ func (b *Backend) MakeRuns(t *Test) (runs []*Run) { if !b.includeTest(t) { return runs } + maxSize := fs.SizeSuffix(0) + if b.MaxFile != "" { + if err := maxSize.Set(b.MaxFile); err != nil { + log.Printf("Invalid maxfile value %q: %v", b.MaxFile, err) + } + } fastlists := []bool{false} if b.FastList && t.FastList { fastlists = append(fastlists, true) @@ -81,6 +88,7 @@ func (b *Backend) MakeRuns(t *Test) (runs []*Run) { NoRetries: t.NoRetries, OneOnly: b.OneOnly, NoBinary: t.NoBinary, + SizeLimit: int64(maxSize), Ignore: ignore, } if t.AddBackend { diff --git a/fstest/test_all/run.go b/fstest/test_all/run.go index 45ddb973b..bc684553b 100644 --- a/fstest/test_all/run.go +++ b/fstest/test_all/run.go @@ -16,6 +16,7 @@ import ( "regexp" "runtime" "sort" + "strconv" "strings" "sync" "time" @@ -43,6 +44,7 @@ type Run struct { NoRetries bool // don't retry if set OneOnly bool // only run test for this backend at once NoBinary bool // set to not build a binary + SizeLimit int64 // maximum test file size Ignore map[string]struct{} // Internals cmdLine []string @@ -339,6 +341,9 @@ func (r *Run) Init() { if r.Short { r.cmdLine = append(r.cmdLine, "-short") } + if r.SizeLimit > 0 { + r.cmdLine = append(r.cmdLine, "-size-limit", strconv.FormatInt(r.SizeLimit, 10)) + } r.cmdString = toShell(r.cmdLine) }