fstests: re-arrange backend integration tests so they can be retried

Before this change backend integration tests depended on each other,
so tests could not be retried.

After this change we nest tests to ensure that tests are provided with
the starting state they expect.

Tell the integration test runner that it can retry backend tests also.

This also includes bin/test_independence.go which runs each test
individually for a backend to prove that they are independent.
This commit is contained in:
Nick Craig-Wood 2018-11-22 17:43:18 +00:00
parent e969505ae4
commit 31e2ce03c3
4 changed files with 1147 additions and 1072 deletions

View File

@ -243,10 +243,19 @@ func (f *Fs) InternalTestDocumentLink(t *testing.T) {
}
func (f *Fs) InternalTest(t *testing.T) {
t.Run("DocumentImport", f.InternalTestDocumentImport)
t.Run("DocumentUpdate", f.InternalTestDocumentUpdate)
t.Run("DocumentExport", f.InternalTestDocumentExport)
t.Run("DocumentLink", f.InternalTestDocumentLink)
// These tests all depend on each other so run them as nested tests
t.Run("DocumentImport", func(t *testing.T) {
f.InternalTestDocumentImport(t)
t.Run("DocumentUpdate", func(t *testing.T) {
f.InternalTestDocumentUpdate(t)
t.Run("DocumentExport", func(t *testing.T) {
f.InternalTestDocumentExport(t)
t.Run("DocumentLink", func(t *testing.T) {
f.InternalTestDocumentLink(t)
})
})
})
})
}
var _ fstests.InternalTester = (*Fs)(nil)

59
bin/test_independence.go Normal file
View File

@ -0,0 +1,59 @@
// +build ignore
// Test that the tests in the suite passed in are independent
package main
import (
"flag"
"log"
"os"
"os/exec"
"regexp"
)
var matchLine = regexp.MustCompile(`(?m)^=== RUN\s*(TestIntegration/\S*)\s*$`)
// run the test pass in and grep out the test names
func findTests(packageToTest string) (tests []string) {
cmd := exec.Command("go", "test", "-v", packageToTest)
out, err := cmd.CombinedOutput()
if err != nil {
_, _ = os.Stderr.Write(out)
log.Fatal(err)
}
results := matchLine.FindAllSubmatch(out, -1)
if results == nil {
log.Fatal("No tests found")
}
for _, line := range results {
tests = append(tests, string(line[1]))
}
return tests
}
// run the test passed in with the -run passed in
func runTest(packageToTest string, testName string) {
cmd := exec.Command("go", "test", "-v", packageToTest, "-run", "^"+testName+"$")
out, err := cmd.CombinedOutput()
if err != nil {
log.Printf("%s FAILED ------------------", testName)
_, _ = os.Stderr.Write(out)
log.Printf("%s FAILED ------------------", testName)
} else {
log.Printf("%s OK", testName)
}
}
func main() {
flag.Parse()
args := flag.Args()
if len(args) != 1 {
log.Fatalf("Syntax: %s <test_to_run>", os.Args[0])
}
packageToTest := args[0]
testNames := findTests(packageToTest)
// fmt.Printf("%s\n", testNames)
for _, testName := range testNames {
runTest(packageToTest, testName)
}
}

File diff suppressed because it is too large Load Diff

View File

@ -1,7 +1,6 @@
tests:
- path: backend
addbackend: true
noretries: true
nobinary: true
- path: fs/operations
subdir: true