fstest/test_all: rework integration tests to improve output

- Make integration tests use a config file
- Output individual logs for each test
- Make HTML report and open browser
- Optionally email and upload results
This commit is contained in:
Nick Craig-Wood 2018-09-29 14:48:29 +01:00
parent a3c55462a8
commit f97c4c8d9d
8 changed files with 986 additions and 421 deletions

View File

@ -123,6 +123,13 @@ but they can be run against any of the remotes.
cd fs/operations
go test -v -remote TestDrive:
If you want to use the integration test framework to run these tests
all together with an HTML report and test retries then from the
project root:
go install github.com/ncw/rclone/fstest/test_all
test_all -backend drive
If you want to run all the integration tests against all the remotes,
then change into the project root and run
@ -343,7 +350,7 @@ Unit tests
Integration tests
* Add your fs to `fstest/test_all/test_all.go`
* Add your backend to `fstest/test_all/config.yaml`
* Make sure integration tests pass with
* `cd fs/operations`
* `go test -v -remote TestRemote:`

View File

@ -51,9 +51,8 @@ version:
# Full suite of integration tests
test: rclone
go install github.com/ncw/rclone/fstest/test_all
-go test -v -count 1 -timeout 20m $(BUILDTAGS) $(GO_FILES) 2>&1 | tee test.log
-test_all github.com/ncw/rclone/fs/operations github.com/ncw/rclone/fs/sync 2>&1 | tee fs/test_all.log
@echo "Written logs in test.log and fs/test_all.log"
-test_all 2>&1 | tee test_all.log
@echo "Written logs in test_all.log"
# Quick test
quicktest:

60
fstest/test_all/clean.go Normal file
View File

@ -0,0 +1,60 @@
// Clean the left over test files
package main
import (
"log"
"regexp"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fs/list"
"github.com/ncw/rclone/fs/operations"
)
// MatchTestRemote matches the remote names used for testing (copied
// from fstest/fstest.go so we don't have to import that and get all
// its flags)
var MatchTestRemote = regexp.MustCompile(`^rclone-test-[abcdefghijklmnopqrstuvwxyz0123456789]{24}$`)
// cleanFs runs a single clean fs for left over directories
func cleanFs(remote string) error {
f, err := fs.NewFs(remote)
if err != nil {
return err
}
entries, err := list.DirSorted(f, true, "")
if err != nil {
return err
}
return entries.ForDirError(func(dir fs.Directory) error {
dirPath := dir.Remote()
fullPath := remote + dirPath
if MatchTestRemote.MatchString(dirPath) {
if *dryRun {
log.Printf("Not Purging %s - -dry-run", fullPath)
return nil
}
log.Printf("Purging %s", fullPath)
dir, err := fs.NewFs(fullPath)
if err != nil {
return err
}
return operations.Purge(dir, "")
}
return nil
})
}
// cleanRemotes cleans the list of remotes passed in
func cleanRemotes(remotes []string) error {
var lastError error
for _, remote := range remotes {
log.Printf("%q - Cleaning", remote)
err := cleanFs(remote)
if err != nil {
lastError = err
log.Printf("Failed to purge %q: %v", remote, err)
}
}
return lastError
}

159
fstest/test_all/config.go Normal file
View File

@ -0,0 +1,159 @@
// Config handling
package main
import (
"io/ioutil"
"log"
"path"
"github.com/pkg/errors"
"gopkg.in/yaml.v2"
)
// Test describes an integration test to run with `go test`
type Test struct {
Path string // path to the source directory
SubDir bool // if it is possible to add -sub-dir to tests
FastList bool // if it is possible to add -fast-list to tests
AddBackend bool // set if Path needs the current backend appending
NoRetries bool // set if no retries should be performed
}
// Backend describes a backend test
//
// FIXME make bucket based remotes set sub-dir automatically???
type Backend struct {
Backend string // name of the backend directory
Remote string // name of the test remote
SubDir bool // set to test with -sub-dir
FastList bool // set to test with -fast-list
}
// MakeRuns creates Run objects the Backend and Test
//
// There can be several created, one for each combination of SubDir
// and FastList
func (b *Backend) MakeRuns(t *Test) (runs []*Run) {
subdirs := []bool{false}
if b.SubDir && t.SubDir {
subdirs = append(subdirs, true)
}
fastlists := []bool{false}
if b.FastList && t.FastList {
fastlists = append(fastlists, true)
}
for _, subdir := range subdirs {
for _, fastlist := range fastlists {
run := &Run{
Remote: b.Remote,
Backend: b.Backend,
Path: t.Path,
SubDir: subdir,
FastList: fastlist,
NoRetries: t.NoRetries,
}
if t.AddBackend {
run.Path = path.Join(run.Path, b.Backend)
}
runs = append(runs, run)
}
}
return runs
}
// Config describes the config for this program
type Config struct {
Tests []Test
Backends []Backend
}
// NewConfig reads the config file
func NewConfig(configFile string) (*Config, error) {
d, err := ioutil.ReadFile(configFile)
if err != nil {
return nil, errors.Wrap(err, "failed to read config file")
}
config := &Config{}
err = yaml.Unmarshal(d, &config)
if err != nil {
return nil, errors.Wrap(err, "failed to parse config file")
}
// d, err = yaml.Marshal(&config)
// if err != nil {
// log.Fatalf("error: %v", err)
// }
// fmt.Printf("--- m dump:\n%s\n\n", string(d))
return config, nil
}
// MakeRuns makes Run objects for each combination of Backend and Test
// in the config
func (c *Config) MakeRuns() (runs []*Run) {
for _, backend := range c.Backends {
for _, test := range c.Tests {
runs = append(runs, backend.MakeRuns(&test)...)
}
}
return runs
}
// Filter the Backends with the remotes passed in.
//
// If no backend is found with a remote is found then synthesize one
func (c *Config) filterBackendsByRemotes(remotes []string) {
var newBackends []Backend
for _, name := range remotes {
found := false
for i := range c.Backends {
if c.Backends[i].Remote == name {
newBackends = append(newBackends, c.Backends[i])
found = true
}
}
if !found {
log.Printf("Remote %q not found - inserting with default flags", name)
newBackends = append(newBackends, Backend{Remote: name})
}
}
c.Backends = newBackends
}
// Filter the Backends with the backendNames passed in
func (c *Config) filterBackendsByBackends(backendNames []string) {
var newBackends []Backend
for _, name := range backendNames {
for i := range c.Backends {
if c.Backends[i].Backend == name {
newBackends = append(newBackends, c.Backends[i])
}
}
}
c.Backends = newBackends
}
// Filter the incoming tests into the backends selected
func (c *Config) filterTests(paths []string) {
var newTests []Test
for _, path := range paths {
for i := range c.Tests {
if c.Tests[i].Path == path {
newTests = append(newTests, c.Tests[i])
}
}
}
c.Tests = newTests
}
// Remotes returns the unique remotes
func (c *Config) Remotes() (remotes []string) {
found := map[string]struct{}{}
for _, backend := range c.Backends {
if _, ok := found[backend.Remote]; ok {
continue
}
remotes = append(remotes, backend.Remote)
found[backend.Remote] = struct{}{}
}
return remotes
}

107
fstest/test_all/config.yaml Normal file
View File

@ -0,0 +1,107 @@
tests:
- path: backend
addbackend: true
noretries: true
- path: fs/operations
subdir: true
fastlist: true
- path: fs/sync
subdir: true
fastlist: true
backends:
# - backend: "amazonclouddrive"
# remote: "TestAmazonCloudDrive:"
# subdir: false
# fastlist: false
- backend: "b2"
remote: "TestB2:"
subdir: true
fastlist: true
- backend: "crypt"
remote: "TestCryptDrive:"
subdir: false
fastlist: true
- backend: "crypt"
remote: "TestCryptSwift:"
subdir: false
fastlist: false
- backend: "drive"
remote: "TestDrive:"
subdir: false
fastlist: true
- backend: "dropbox"
remote: "TestDropbox:"
subdir: false
fastlist: false
- backend: "googlecloudstorage"
remote: "TestGoogleCloudStorage:"
subdir: true
fastlist: true
- backend: "hubic"
remote: "TestHubic:"
subdir: false
fastlist: false
- backend: "jottacloud"
remote: "TestJottacloud:"
subdir: false
fastlist: true
- backend: "onedrive"
remote: "TestOneDrive:"
subdir: false
fastlist: false
- backend: "s3"
remote: "TestS3:"
subdir: true
fastlist: true
- backend: "sftp"
remote: "TestSftp:"
subdir: false
fastlist: false
- backend: "swift"
remote: "TestSwift:"
subdir: true
fastlist: true
- backend: "yandex"
remote: "TestYandex:"
subdir: false
fastlist: false
- backend: "ftp"
remote: "TestFTP:"
subdir: false
fastlist: false
- backend: "box"
remote: "TestBox:"
subdir: false
fastlist: false
- backend: "qingstor"
remote: "TestQingStor:"
subdir: false
fastlist: false
- backend: "azureblob"
remote: "TestAzureBlob:"
subdir: true
fastlist: true
- backend: "pcloud"
remote: "TestPcloud:"
subdir: false
fastlist: false
- backend: "webdav"
remote: "TestWebdav:"
subdir: false
fastlist: false
- backend: "cache"
remote: "TestCache:"
subdir: false
fastlist: false
- backend: "mega"
remote: "TestMega:"
subdir: false
fastlist: false
- backend: "opendrive"
remote: "TestOpenDrive:"
subdir: false
fastlist: false
- backend: "union"
remote: "TestUnion:"
subdir: false
fastlist: false

260
fstest/test_all/report.go Normal file
View File

@ -0,0 +1,260 @@
package main
import (
"fmt"
"html/template"
"io/ioutil"
"log"
"os"
"os/exec"
"path"
"sort"
"time"
"github.com/ncw/rclone/fs"
"github.com/skratchdot/open-golang/open"
)
const timeFormat = "2006-01-02-150405"
// Report holds the info to make a report on a series of test runs
type Report struct {
LogDir string // output directory for logs and report
StartTime time.Time // time started
DateTime string // directory name for output
Duration time.Duration // time the run took
Failed Runs // failed runs
Passed Runs // passed runs
Runs []ReportRun // runs to report
Version string // rclone version
Previous string // previous test name if known
IndexHTML string // path to the index.html file
URL string // online version
}
// ReportRun is used in the templates to report on a test run
type ReportRun struct {
Name string
Runs Runs
}
// NewReport initialises and returns a Report
func NewReport() *Report {
r := &Report{
StartTime: time.Now(),
Version: fs.Version,
}
r.DateTime = r.StartTime.Format(timeFormat)
// Find previous log directory if possible
names, err := ioutil.ReadDir(*outputDir)
if err == nil && len(names) > 0 {
r.Previous = names[len(names)-1].Name()
}
// Create output directory for logs and report
r.LogDir = path.Join(*outputDir, r.DateTime)
err = os.MkdirAll(r.LogDir, 0777)
if err != nil {
log.Fatalf("Failed to make log directory: %v", err)
}
// Online version
r.URL = *urlBase + r.DateTime + "/index.html"
return r
}
// End should be called when the tests are complete
func (r *Report) End() {
r.Duration = time.Since(r.StartTime)
sort.Sort(r.Failed)
sort.Sort(r.Passed)
r.Runs = []ReportRun{
{Name: "Failed", Runs: r.Failed},
{Name: "Passed", Runs: r.Passed},
}
}
// AllPassed returns true if there were no failed tests
func (r *Report) AllPassed() bool {
return len(r.Failed) == 0
}
// RecordResult should be called with a Run when it has finished to be
// recorded into the Report
func (r *Report) RecordResult(t *Run) {
if !t.passed() {
r.Failed = append(r.Failed, t)
} else {
r.Passed = append(r.Passed, t)
}
}
// Title returns a human readable summary title for the Report
func (r *Report) Title() string {
if r.AllPassed() {
return fmt.Sprintf("PASS: All tests finished OK in %v", r.Duration)
}
return fmt.Sprintf("FAIL: %d tests failed in %v", len(r.Failed), r.Duration)
}
// LogSummary writes the summary to the log file
func (r *Report) LogSummary() {
log.Printf("Logs in %q", r.LogDir)
// Summarise results
log.Printf("SUMMARY")
log.Println(r.Title())
if !r.AllPassed() {
for _, t := range r.Failed {
log.Printf(" * %s", toShell(t.nextCmdLine()))
log.Printf(" * Failed tests: %v", t.failedTests)
}
}
}
// LogHTML writes the summary to index.html in LogDir
func (r *Report) LogHTML() {
r.IndexHTML = path.Join(r.LogDir, "index.html")
out, err := os.Create(r.IndexHTML)
if err != nil {
log.Fatalf("Failed to open index.html: %v", err)
}
defer func() {
err := out.Close()
if err != nil {
log.Fatalf("Failed to close index.html: %v", err)
}
}()
err = reportTemplate.Execute(out, r)
if err != nil {
log.Fatalf("Failed to execute template: %v", err)
}
_ = open.Start("file://" + r.IndexHTML)
}
var reportHTML = `<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>{{ .Title }}</title>
<style>
table {
border-collapse: collapse;
border-spacing: 0;
border: 1px solid #ddd;
}
table.tests {
width: 100%;
}
table, th, td {
border: 1px solid black;
}
.Failed {
color: red;
}
.Passed {
color: green;
}
.false {
font-weight: lighter;
}
.true {
font-weight: bold;
}
th, td {
text-align: left;
padding: 4px;
}
tr:nth-child(even) {
background-color: #f2f2f2
}
</style>
</head>
<body>
<h1>{{ .Title }}</h1>
<table>
<tr><th>Version</th><td>{{ .Version }}</td></tr>
<tr><th>Date</th><td>{{ .DateTime}} [<a href="{{ .URL }}">online</a>]</td></tr>
<tr><th>Duration</th><td>{{ .Duration }}</td></tr>
{{ if .Previous}}<tr><th>Previous</th><td><a href="../{{ .Previous }}/index.html">{{ .Previous }}</a></td></tr>{{ end }}
<tr><th>Up</th><td><a href="../">Older Tests</a></td></tr>
</table>
{{ range .Runs }}
{{ if .Runs }}
<h2 class="{{ .Name }}">{{ .Name }}: {{ len .Runs }}</h2>
<table class="{{ .Name }} tests">
<tr>
<th>Backend</th>
<th>Remote</th>
<th>Test</th>
<th>SubDir</th>
<th>FastList</th>
<th>Failed</th>
<th>Logs</th>
</tr>
{{ $prevBackend := "" }}
{{ $prevRemote := "" }}
{{ range .Runs}}
<tr>
<td>{{ if ne $prevBackend .Backend }}{{ .Backend }}{{ end }}{{ $prevBackend = .Backend }}</td>
<td>{{ if ne $prevRemote .Remote }}{{ .Remote }}{{ end }}{{ $prevRemote = .Remote }}</td>
<td>{{ .Path }}</td>
<td><span class="{{ .SubDir }}">{{ .SubDir }}</span></td>
<td><span class="{{ .FastList }}">{{ .FastList }}</span></td>
<td>{{ .FailedTests }}</td>
<td>{{ range $i, $v := .Logs }}<a href="{{ $v }}">#{{ $i }}</a> {{ end }}</td>
</tr>
{{ end }}
</table>
{{ end }}
{{ end }}
</body>
</html>
`
var reportTemplate = template.Must(template.New("Report").Parse(reportHTML))
// EmailHTML sends the summary report to the email address supplied
func (r *Report) EmailHTML() {
if *emailReport == "" || r.IndexHTML == "" {
return
}
log.Printf("Sending email summary to %q", *emailReport)
cmdLine := []string{"mail", "-a", "Content-Type: text/html", *emailReport, "-s", "rclone integration tests: " + r.Title()}
cmd := exec.Command(cmdLine[0], cmdLine[1:]...)
in, err := os.Open(r.IndexHTML)
if err != nil {
log.Fatalf("Failed to open index.html: %v", err)
}
cmd.Stdin = in
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
err = cmd.Run()
if err != nil {
log.Fatalf("Failed to send email: %v", err)
}
_ = in.Close()
}
// Upload uploads a copy of the report online
func (r *Report) Upload() {
if *uploadPath == "" || r.IndexHTML == "" {
return
}
dst := path.Join(*uploadPath, r.DateTime)
log.Printf("Uploading results to %q", dst)
cmdLine := []string{"rclone", "copy", "-v", r.LogDir, dst}
cmd := exec.Command(cmdLine[0], cmdLine[1:]...)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
err := cmd.Run()
if err != nil {
log.Fatalf("Failed to upload results: %v", err)
}
}

318
fstest/test_all/run.go Normal file
View File

@ -0,0 +1,318 @@
// Run a test
package main
import (
"bytes"
"fmt"
"go/build"
"io"
"log"
"os"
"os/exec"
"path"
"regexp"
"runtime"
"strings"
"time"
"github.com/ncw/rclone/fs"
)
const testBase = "github.com/ncw/rclone/"
// Run holds info about a running test
//
// A run just runs one command line, but it can be run multiple times
// if retries are needed.
type Run struct {
// Config
Remote string // name of the test remote
Backend string // name of the backend
Path string // path to the source directory
SubDir bool // add -sub-dir to tests
FastList bool // add -fast-list to tests
NoRetries bool // don't retry if set
// Internals
cmdLine []string
cmdString string
try int
err error
output []byte
failedTests []string
runFlag string
logDir string // directory to place the logs
trialName string // name/log file name of current trial
trialNames []string // list of all the trials
}
// Runs records multiple Run objects
type Runs []*Run
// Sort interface
func (rs Runs) Len() int { return len(rs) }
func (rs Runs) Swap(i, j int) { rs[i], rs[j] = rs[j], rs[i] }
func (rs Runs) Less(i, j int) bool {
a, b := rs[i], rs[j]
if a.Backend < b.Backend {
return true
} else if a.Backend > b.Backend {
return false
}
if a.Remote < b.Remote {
return true
} else if a.Remote > b.Remote {
return false
}
if a.Path < b.Path {
return true
} else if a.Path > b.Path {
return false
}
if !a.SubDir && b.SubDir {
return true
} else if a.SubDir && !b.SubDir {
return false
}
if !a.FastList && b.FastList {
return true
} else if a.FastList && !b.FastList {
return false
}
return false
}
// dumpOutput prints the error output
func (r *Run) dumpOutput() {
log.Println("------------------------------------------------------------")
log.Printf("---- %q ----", r.cmdString)
log.Println(string(r.output))
log.Println("------------------------------------------------------------")
}
var failRe = regexp.MustCompile(`(?m)^--- FAIL: (Test\w*) \(`)
// findFailures looks for all the tests which failed
func (r *Run) findFailures() {
oldFailedTests := r.failedTests
r.failedTests = nil
for _, matches := range failRe.FindAllSubmatch(r.output, -1) {
r.failedTests = append(r.failedTests, string(matches[1]))
}
if len(r.failedTests) != 0 {
r.runFlag = "^(" + strings.Join(r.failedTests, "|") + ")$"
} else {
r.runFlag = ""
}
if r.passed() && len(r.failedTests) != 0 {
log.Printf("%q - Expecting no errors but got: %v", r.cmdString, r.failedTests)
r.dumpOutput()
} else if !r.passed() && len(r.failedTests) == 0 {
log.Printf("%q - Expecting errors but got none: %v", r.cmdString, r.failedTests)
r.dumpOutput()
r.failedTests = oldFailedTests
}
}
// nextCmdLine returns the next command line
func (r *Run) nextCmdLine() []string {
cmdLine := r.cmdLine
if r.runFlag != "" {
cmdLine = append(cmdLine, "-test.run", r.runFlag)
}
return cmdLine
}
// trial runs a single test
func (r *Run) trial() {
cmdLine := r.nextCmdLine()
cmdString := toShell(cmdLine)
msg := fmt.Sprintf("%q - Starting (try %d/%d)", cmdString, r.try, *maxTries)
log.Println(msg)
logName := path.Join(r.logDir, r.trialName)
out, err := os.Create(logName)
if err != nil {
log.Fatalf("Couldn't create log file: %v", err)
}
defer func() {
err := out.Close()
if err != nil {
log.Fatalf("Failed to close log file: %v", err)
}
}()
_, _ = fmt.Fprintln(out, msg)
// Early exit if --try-run
if *dryRun {
log.Printf("Not executing as --dry-run: %v", cmdLine)
_, _ = fmt.Fprintln(out, "--dry-run is set - not running")
return
}
// Internal buffer
var b bytes.Buffer
multiOut := io.MultiWriter(out, &b)
cmd := exec.Command(cmdLine[0], cmdLine[1:]...)
cmd.Stderr = multiOut
cmd.Stdout = multiOut
start := time.Now()
r.err = cmd.Run()
r.output = b.Bytes()
duration := time.Since(start)
r.findFailures()
if r.passed() {
msg = fmt.Sprintf("%q - Finished OK in %v (try %d/%d)", cmdString, duration, r.try, *maxTries)
} else {
msg = fmt.Sprintf("%q - Finished ERROR in %v (try %d/%d): %v: Failed %v", cmdString, duration, r.try, *maxTries, r.err, r.failedTests)
}
log.Println(msg)
_, _ = fmt.Fprintln(out, msg)
}
// passed returns true if the test passed
func (r *Run) passed() bool {
return r.err == nil
}
// GOPATH returns the current GOPATH
func GOPATH() string {
gopath := os.Getenv("GOPATH")
if gopath == "" {
gopath = build.Default.GOPATH
}
return gopath
}
// BinaryName turns a package name into a binary name
func (r *Run) BinaryName() string {
binary := path.Base(r.Path) + ".test"
if runtime.GOOS == "windows" {
binary += ".exe"
}
return binary
}
// BinaryPath turns a package name into a binary path
func (r *Run) BinaryPath() string {
return path.Join(r.Path, r.BinaryName())
}
// PackagePath returns the path to the package
func (r *Run) PackagePath() string {
return path.Join(GOPATH(), "src", r.Path)
}
// Chdir into the package directory
func (r *Run) Chdir() {
err := os.Chdir(r.PackagePath())
if err != nil {
log.Fatalf("Failed to chdir to package %q: %v", r.Path, err)
}
}
// MakeTestBinary makes the binary we will run
func (r *Run) MakeTestBinary() {
binary := r.BinaryPath()
binaryName := r.BinaryName()
log.Printf("%s: Making test binary %q", r.Path, binaryName)
cmdLine := []string{"go", "test", "-c", "-o", binary, testBase + r.Path}
if *dryRun {
log.Printf("Not executing: %v", cmdLine)
return
}
err := exec.Command(cmdLine[0], cmdLine[1:]...).Run()
if err != nil {
log.Fatalf("Failed to make test binary: %v", err)
}
if _, err := os.Stat(binary); err != nil {
log.Fatalf("Couldn't find test binary %q", binary)
}
}
// RemoveTestBinary removes the binary made in makeTestBinary
func (r *Run) RemoveTestBinary() {
if *dryRun {
return
}
binary := r.BinaryPath()
err := os.Remove(binary) // Delete the binary when finished
if err != nil {
log.Printf("Error removing test binary %q: %v", binary, err)
}
}
// Name returns the run name as a file name friendly string
func (r *Run) Name() string {
ns := []string{
r.Backend,
strings.Replace(r.Path, "/", ".", -1),
r.Remote,
}
if r.SubDir {
ns = append(ns, "subdir")
}
if r.FastList {
ns = append(ns, "fastlist")
}
ns = append(ns, fmt.Sprintf("%d", r.try))
s := strings.Join(ns, "-")
s = strings.Replace(s, ":", "", -1)
return s
}
// Init the Run
func (r *Run) Init() {
binary := r.BinaryPath()
r.cmdLine = []string{binary, "-test.v", "-test.timeout", timeout.String(), "-remote", r.Remote}
r.try = 1
if *verbose {
r.cmdLine = append(r.cmdLine, "-verbose")
fs.Config.LogLevel = fs.LogLevelDebug
}
if *runOnly != "" {
r.cmdLine = append(r.cmdLine, "-test.run", *runOnly)
}
if r.SubDir {
r.cmdLine = append(r.cmdLine, "-subdir")
}
if r.FastList {
r.cmdLine = append(r.cmdLine, "-fast-list")
}
r.cmdString = toShell(r.cmdLine)
}
// Logs returns all the log names
func (r *Run) Logs() []string {
return r.trialNames
}
// FailedTests returns the failed tests as a comma separated string, limiting the number
func (r *Run) FailedTests() string {
const maxTests = 5
ts := r.failedTests
if len(ts) > maxTests {
ts = ts[:maxTests:maxTests]
ts = append(ts, fmt.Sprintf("… (%d more)", len(r.failedTests)-maxTests))
}
return strings.Join(ts, ", ")
}
// Run runs all the trials for this test
func (r *Run) Run(logDir string, result chan<- *Run) {
r.Init()
r.logDir = logDir
for r.try = 1; r.try <= *maxTries; r.try++ {
r.trialName = r.Name() + ".txt"
r.trialNames = append(r.trialNames, r.trialName)
log.Printf("Starting run with log %q", r.trialName)
r.trial()
if r.passed() || r.NoRetries {
break
}
}
if !r.passed() {
r.dumpOutput()
}
result <- r
}

View File

@ -4,24 +4,22 @@
// See the `test` target in the Makefile.
package main
/* FIXME
Make TesTrun have a []string of flags to try - that then makes it generic
*/
import (
"flag"
"go/build"
"log"
"os"
"os/exec"
"path"
"regexp"
"runtime"
"strings"
"time"
_ "github.com/ncw/rclone/backend/all" // import all fs
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fs/config"
"github.com/ncw/rclone/fs/list"
"github.com/ncw/rclone/fs/operations"
"github.com/ncw/rclone/fstest"
)
type remoteConfig struct {
@ -31,218 +29,23 @@ type remoteConfig struct {
}
var (
remotes = []remoteConfig{
// {
// Name: "TestAmazonCloudDrive:",
// SubDir: false,
// FastList: false,
// },
{
Name: "TestB2:",
SubDir: true,
FastList: true,
},
{
Name: "TestCryptDrive:",
SubDir: false,
FastList: true,
},
{
Name: "TestCryptSwift:",
SubDir: false,
FastList: false,
},
{
Name: "TestDrive:",
SubDir: false,
FastList: true,
},
{
Name: "TestDropbox:",
SubDir: false,
FastList: false,
},
{
Name: "TestGoogleCloudStorage:",
SubDir: true,
FastList: true,
},
{
Name: "TestHubic:",
SubDir: false,
FastList: false,
},
{
Name: "TestJottacloud:",
SubDir: false,
FastList: true,
},
{
Name: "TestOneDrive:",
SubDir: false,
FastList: false,
},
{
Name: "TestS3:",
SubDir: true,
FastList: true,
},
{
Name: "TestSftp:",
SubDir: false,
FastList: false,
},
{
Name: "TestSwift:",
SubDir: true,
FastList: true,
},
{
Name: "TestYandex:",
SubDir: false,
FastList: false,
},
{
Name: "TestFTP:",
SubDir: false,
FastList: false,
},
{
Name: "TestBox:",
SubDir: false,
FastList: false,
},
{
Name: "TestQingStor:",
SubDir: false,
FastList: false,
},
{
Name: "TestAzureBlob:",
SubDir: true,
FastList: true,
},
{
Name: "TestPcloud:",
SubDir: false,
FastList: false,
},
{
Name: "TestWebdav:",
SubDir: false,
FastList: false,
},
{
Name: "TestCache:",
SubDir: false,
FastList: false,
},
{
Name: "TestMega:",
SubDir: false,
FastList: false,
},
{
Name: "TestOpenDrive:",
SubDir: false,
FastList: false,
},
{
Name: "TestUnion:",
SubDir: false,
FastList: false,
},
}
// Flags
maxTries = flag.Int("maxtries", 5, "Number of times to try each test")
runTests = flag.String("remotes", "", "Comma separated list of remotes to test, eg 'TestSwift:,TestS3'")
testRemotes = flag.String("remotes", "", "Comma separated list of remotes to test, eg 'TestSwift:,TestS3'")
testBackends = flag.String("backends", "", "Comma separated list of backends to test, eg 's3,googlecloudstorage")
testTests = flag.String("tests", "", "Comma separated list of tests to test, eg 'fs/sync,fs/operations'")
clean = flag.Bool("clean", false, "Instead of testing, clean all left over test directories")
runOnly = flag.String("run", "", "Run only those tests matching the regexp supplied")
timeout = flag.Duration("timeout", 30*time.Minute, "Maximum time to run each test for before giving up")
configFile = flag.String("config", "fstest/test_all/config.yaml", "Path to config file")
outputDir = flag.String("output", path.Join(os.TempDir(), "rclone-integration-tests"), "Place to store results")
emailReport = flag.String("email", "", "Set to email the report to the address supplied")
dryRun = flag.Bool("dry-run", false, "Print commands which would be executed only")
urlBase = flag.String("url-base", "https://pub.rclone.org/integration-tests/", "Base for the online version")
uploadPath = flag.String("upload", "", "Set this to an rclone path to upload the results here")
verbose = flag.Bool("verbose", false, "Set to enable verbose logging in the tests")
)
// test holds info about a running test
type test struct {
pkg string
remote string
subdir bool
cmdLine []string
cmdString string
try int
err error
output []byte
failedTests []string
runFlag string
}
// newTest creates a new test
func newTest(pkg, remote string, subdir bool, fastlist bool) *test {
binary := pkgBinary(pkg)
t := &test{
pkg: pkg,
remote: remote,
subdir: subdir,
cmdLine: []string{binary, "-test.timeout", timeout.String(), "-remote", remote},
try: 1,
}
if *fstest.Verbose {
t.cmdLine = append(t.cmdLine, "-test.v")
fs.Config.LogLevel = fs.LogLevelDebug
}
if *runOnly != "" {
t.cmdLine = append(t.cmdLine, "-test.run", *runOnly)
}
if subdir {
t.cmdLine = append(t.cmdLine, "-subdir")
}
if fastlist {
t.cmdLine = append(t.cmdLine, "-fast-list")
}
t.cmdString = toShell(t.cmdLine)
return t
}
// dumpOutput prints the error output
func (t *test) dumpOutput() {
log.Println("------------------------------------------------------------")
log.Printf("---- %q ----", t.cmdString)
log.Println(string(t.output))
log.Println("------------------------------------------------------------")
}
var failRe = regexp.MustCompile(`(?m)^--- FAIL: (Test\w*) \(`)
// findFailures looks for all the tests which failed
func (t *test) findFailures() {
oldFailedTests := t.failedTests
t.failedTests = nil
for _, matches := range failRe.FindAllSubmatch(t.output, -1) {
t.failedTests = append(t.failedTests, string(matches[1]))
}
if len(t.failedTests) != 0 {
t.runFlag = "^(" + strings.Join(t.failedTests, "|") + ")$"
} else {
t.runFlag = ""
}
if t.passed() && len(t.failedTests) != 0 {
log.Printf("%q - Expecting no errors but got: %v", t.cmdString, t.failedTests)
t.dumpOutput()
} else if !t.passed() && len(t.failedTests) == 0 {
log.Printf("%q - Expecting errors but got none: %v", t.cmdString, t.failedTests)
t.dumpOutput()
t.failedTests = oldFailedTests
}
}
// nextCmdLine returns the next command line
func (t *test) nextCmdLine() []string {
cmdLine := t.cmdLine
if t.runFlag != "" {
cmdLine = append(cmdLine, "-test.run", t.runFlag)
}
return cmdLine
}
// if matches then is definitely OK in the shell
var shellOK = regexp.MustCompile("^[A-Za-z0-9./_:-]+$")
@ -261,181 +64,53 @@ func toShell(args []string) (result string) {
return result
}
// trial runs a single test
func (t *test) trial() {
cmdLine := t.nextCmdLine()
cmdString := toShell(cmdLine)
log.Printf("%q - Starting (try %d/%d)", cmdString, t.try, *maxTries)
cmd := exec.Command(cmdLine[0], cmdLine[1:]...)
start := time.Now()
t.output, t.err = cmd.CombinedOutput()
duration := time.Since(start)
t.findFailures()
if t.passed() {
log.Printf("%q - Finished OK in %v (try %d/%d)", cmdString, duration, t.try, *maxTries)
} else {
log.Printf("%q - Finished ERROR in %v (try %d/%d): %v: Failed %v", cmdString, duration, t.try, *maxTries, t.err, t.failedTests)
}
}
// cleanFs runs a single clean fs for left over directories
func (t *test) cleanFs() error {
f, err := fs.NewFs(t.remote)
if err != nil {
return err
}
entries, err := list.DirSorted(f, true, "")
if err != nil {
return err
}
return entries.ForDirError(func(dir fs.Directory) error {
remote := dir.Remote()
if fstest.MatchTestRemote.MatchString(remote) {
log.Printf("Purging %s%s", t.remote, remote)
dir, err := fs.NewFs(t.remote + remote)
if err != nil {
return err
}
return operations.Purge(dir, "")
}
return nil
})
}
// clean runs a single clean on a fs for left over directories
func (t *test) clean() {
log.Printf("%q - Starting clean (try %d/%d)", t.remote, t.try, *maxTries)
start := time.Now()
t.err = t.cleanFs()
if t.err != nil {
log.Printf("%q - Failed to purge %v", t.remote, t.err)
}
duration := time.Since(start)
if t.passed() {
log.Printf("%q - Finished OK in %v (try %d/%d)", t.cmdString, duration, t.try, *maxTries)
} else {
log.Printf("%q - Finished ERROR in %v (try %d/%d): %v", t.cmdString, duration, t.try, *maxTries, t.err)
}
}
// passed returns true if the test passed
func (t *test) passed() bool {
return t.err == nil
}
// run runs all the trials for this test
func (t *test) run(result chan<- *test) {
for t.try = 1; t.try <= *maxTries; t.try++ {
if *clean {
if !t.subdir {
t.clean()
}
} else {
t.trial()
}
if t.passed() {
break
}
}
if !t.passed() {
t.dumpOutput()
}
result <- t
}
// GOPATH returns the current GOPATH
func GOPATH() string {
gopath := os.Getenv("GOPATH")
if gopath == "" {
gopath = build.Default.GOPATH
}
return gopath
}
// turn a package name into a binary name
func pkgBinaryName(pkg string) string {
binary := path.Base(pkg) + ".test"
if runtime.GOOS == "windows" {
binary += ".exe"
}
return binary
}
// turn a package name into a binary path
func pkgBinary(pkg string) string {
return path.Join(pkgPath(pkg), pkgBinaryName(pkg))
}
// returns the path to the package
func pkgPath(pkg string) string {
return path.Join(GOPATH(), "src", pkg)
}
// cd into the package directory
func pkgChdir(pkg string) {
err := os.Chdir(pkgPath(pkg))
if err != nil {
log.Fatalf("Failed to chdir to package %q: %v", pkg, err)
}
}
// makeTestBinary makes the binary we will run
func makeTestBinary(pkg string) {
binaryName := pkgBinaryName(pkg)
log.Printf("%s: Making test binary %q", pkg, binaryName)
pkgChdir(pkg)
err := exec.Command("go", "test", "-c", "-o", binaryName).Run()
if err != nil {
log.Fatalf("Failed to make test binary: %v", err)
}
binary := pkgBinary(pkg)
if _, err := os.Stat(binary); err != nil {
log.Fatalf("Couldn't find test binary %q", binary)
}
}
// removeTestBinary removes the binary made in makeTestBinary
func removeTestBinary(pkg string) {
binary := pkgBinary(pkg)
err := os.Remove(binary) // Delete the binary when finished
if err != nil {
log.Printf("Error removing test binary %q: %v", binary, err)
}
}
func main() {
flag.Parse()
packages := flag.Args()
log.Printf("Testing packages: %s", strings.Join(packages, ", "))
if *runTests != "" {
newRemotes := []remoteConfig{}
for _, name := range strings.Split(*runTests, ",") {
for i := range remotes {
if remotes[i].Name == name {
newRemotes = append(newRemotes, remotes[i])
goto found
conf, err := NewConfig(*configFile)
if err != nil {
log.Println("test_all should be run from the root of the rclone source code")
log.Fatal(err)
}
// Filter selection
if *testRemotes != "" {
conf.filterBackendsByRemotes(strings.Split(*testRemotes, ","))
}
log.Printf("Remote %q not found - inserting with default flags", name)
newRemotes = append(newRemotes, remoteConfig{Name: name})
found:
if *testBackends != "" {
conf.filterBackendsByBackends(strings.Split(*testBackends, ","))
}
remotes = newRemotes
if *testTests != "" {
conf.filterTests(strings.Split(*testTests, ","))
}
// Just clean the directories if required
if *clean {
err := cleanRemotes(conf.Remotes())
if err != nil {
log.Fatalf("Failed to clean: %v", err)
}
return
}
var names []string
for _, remote := range remotes {
names = append(names, remote.Name)
for _, remote := range conf.Backends {
names = append(names, remote.Remote)
}
log.Printf("Testing remotes: %s", strings.Join(names, ", "))
start := time.Now()
if *clean {
config.LoadConfig()
packages = []string{"clean"}
} else {
for _, pkg := range packages {
makeTestBinary(pkg)
defer removeTestBinary(pkg)
// Runs we will do for this test
runs := conf.MakeRuns()
// Create Report
report := NewReport()
// Make the test binaries, one per Path found in the tests
done := map[string]struct{}{}
for _, run := range runs {
if _, found := done[run.Path]; !found {
done[run.Path] = struct{}{}
run.MakeTestBinary()
defer run.RemoveTestBinary()
}
}
@ -443,46 +118,26 @@ func main() {
_ = os.Setenv("RCLONE_CACHE_DB_WAIT_TIME", "30m")
// start the tests
results := make(chan *test, 8)
results := make(chan *Run, 8)
awaiting := 0
bools := []bool{false, true}
if *clean {
// Don't run -subdir and -fast-list if -clean
bools = bools[:1]
}
for _, pkg := range packages {
for _, remote := range remotes {
for _, subdir := range bools {
for _, fastlist := range bools {
if (!subdir || subdir && remote.SubDir) && (!fastlist || fastlist && remote.FastList) {
go newTest(pkg, remote.Name, subdir, fastlist).run(results)
for _, run := range runs {
go run.Run(report.LogDir, results)
awaiting++
}
}
}
}
}
// Wait for the tests to finish
var failed []*test
for ; awaiting > 0; awaiting-- {
t := <-results
if !t.passed() {
failed = append(failed, t)
report.RecordResult(t)
}
}
duration := time.Since(start)
// Summarise results
log.Printf("SUMMARY")
if len(failed) == 0 {
log.Printf("PASS: All tests finished OK in %v", duration)
} else {
log.Printf("FAIL: %d tests failed in %v", len(failed), duration)
for _, t := range failed {
log.Printf(" * %s", toShell(t.nextCmdLine()))
log.Printf(" * Failed tests: %v", t.failedTests)
}
// Log and exit
report.End()
report.LogSummary()
report.LogHTML()
report.EmailHTML()
report.Upload()
if !report.AllPassed() {
os.Exit(1)
}
}