Something went wrong. Try again.
forked from tailscale/tailscale
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210// Copyright (c) Tailscale Inc & contributors// SPDX-License-Identifier: BSD-3-Clause
package main_test
import ( "bytes" "os" "path/filepath" "strings" "testing")
// multiPkgDir creates a temporary directory for test packages under// testdata, so the packages are inside this module but ignored by// "./..." patterns if a test ever leaks them.func multiPkgDir(t *testing.T) string { t.Helper() dir, err := os.MkdirTemp("testdata", "multipkg-") if err != nil { t.Fatal(err) } t.Cleanup(func() { os.RemoveAll(dir) }) return dir}
// writePkg writes a test-only package named name under dir with the// given test source and returns its relative package pattern.func writePkg(t *testing.T, dir, name, src string) string { t.Helper() pkgDir := filepath.Join(dir, name) if err := os.MkdirAll(pkgDir, 0o755); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(pkgDir, name+"_test.go"), []byte(src), 0o644); err != nil { t.Fatal(err) } return "./" + filepath.ToSlash(pkgDir)}
// importPath returns the import path of a package written by writePkg.func importPath(pattern string) string { return "tailscale.com/cmd/testwrapper/" + strings.TrimPrefix(pattern, "./")}
// firstPassRuns returns the debug "running ..." lines for first-pass// go test invocations, which unlike retries have no --run flag.func firstPassRuns(out []byte) []string { var runs []string for _, line := range strings.Split(string(out), "\n") { if strings.HasPrefix(line, "running test") && !strings.Contains(line, "--run") { runs = append(runs, line) } } return runs}
const passSrc = `package pass_test
import "testing"
func TestPass(t *testing.T) {}`
const flakySrc = `package flaky_test
import ( "os" "testing")
func TestFlaky(t *testing.T) { if os.Getenv("TS_TESTWRAPPER_ATTEMPT") == "1" { t.Fatal("failing on the first attempt so the wrapper retries") }}`
const failSrc = `package fail_test
import "testing"
func TestAlwaysFails(t *testing.T) { t.Fatal("nope") }`
// TestMultiPackage covers several package patterns on one command line:// the first pass must be a single go test invocation naming all of// them, and results, flake retries, and permanent failures must still be// attributed to the right package.func TestMultiPackage(t *testing.T) { t.Parallel()
dir := multiPkgDir(t) passPkg := writePkg(t, dir, "pass", passSrc) flakyPkg := writePkg(t, dir, "flaky", flakySrc) failPkg := writePkg(t, dir, "fail", failSrc)
cmd := cmdTestwrapper(t, passPkg, flakyPkg, failPkg) cmd.Env = append(cmd.Env, "TS_TESTWRAPPER_DEBUG=1") out, err := cmd.CombinedOutput() if code, ok := errExitCode(err); !ok || code != 1 { t.Fatalf("got exit code %d, want 1 (err: %v) with output:\n%s", code, err, out) }
runs := firstPassRuns(out) if len(runs) != 1 { t.Errorf("got %d first-pass go test invocations, want 1:\n%s\noutput:\n%s", len(runs), strings.Join(runs, "\n"), out) } else { for _, p := range []string{passPkg, flakyPkg, failPkg} { if !strings.Contains(runs[0], " "+p+" ") { t.Errorf("first-pass invocation %q lacks package %q", runs[0], p) } } }
for _, want := range []string{ "ok\t" + importPath(passPkg), "FAIL\t" + importPath(flakyPkg), // first pass fails; the retry is per test "FAIL\t" + importPath(failPkg), "flakytest failures JSON:", "permanent test failures JSON:", } { if !bytes.Contains(out, []byte(want)) { t.Errorf("output lacks %q:\n%s", want, out) } } flakyLine, permLine := jsonLines(out) if !strings.Contains(flakyLine, importPath(flakyPkg)) || !strings.Contains(flakyLine, "TestFlaky") { t.Errorf("flaky failures JSON %q should name %s TestFlaky", flakyLine, importPath(flakyPkg)) } if !strings.Contains(permLine, importPath(failPkg)) || !strings.Contains(permLine, "TestAlwaysFails") { t.Errorf("permanent failures JSON %q should name %s TestAlwaysFails", permLine, importPath(failPkg)) } if strings.Contains(permLine, "TestFlaky") || strings.Contains(flakyLine, "TestAlwaysFails") { t.Errorf("failures attributed to the wrong list:\nflaky: %s\npermanent: %s", flakyLine, permLine) }}
// jsonLines returns the flakytest and permanent failures JSON lines// from testwrapper output, or "" for each that is absent.func jsonLines(out []byte) (flaky, permanent string) { for _, line := range strings.Split(string(out), "\n") { if strings.HasPrefix(line, "flakytest failures JSON:") { flaky = line } if strings.HasPrefix(line, "permanent test failures JSON:") { permanent = line } } return flaky, permanent}
// TestMultiPackageBuildError covers a build error in one package of a// batch: the other packages still run and report, and the wrapper// exits with go test's exit code.func TestMultiPackageBuildError(t *testing.T) { t.Parallel()
dir := multiPkgDir(t) passPkg := writePkg(t, dir, "pass", passSrc) brokenPkg := writePkg(t, dir, "broken", "package broken_test\n\nderp\n")
out, err := cmdTestwrapper(t, passPkg, brokenPkg).CombinedOutput() if code, ok := errExitCode(err); !ok || code != 1 { t.Fatalf("got exit code %d, want 1 (err: %v) with output:\n%s", code, err, out) } for _, want := range []string{ "ok\t" + importPath(passPkg), "expected declaration, found derp", } { if !bytes.Contains(out, []byte(want)) { t.Errorf("output lacks %q:\n%s", want, out) } }}
// TestMultiFile covers several .go file arguments: each is its own// "command-line-arguments" package, so each gets its own first-pass// invocation and retries target the right file.func TestMultiFile(t *testing.T) { t.Parallel()
dir := t.TempDir() fileA := filepath.Join(dir, "a_test.go") fileB := filepath.Join(dir, "b_test.go") if err := os.WriteFile(fileA, []byte(passSrc), 0o644); err != nil { t.Fatal(err) } if err := os.WriteFile(fileB, []byte(flakySrc), 0o644); err != nil { t.Fatal(err) }
cmd := cmdTestwrapper(t, fileA, fileB) cmd.Env = append(cmd.Env, "TS_TESTWRAPPER_DEBUG=1") out, err := cmd.CombinedOutput() if err != nil { t.Fatalf("testwrapper: %v with output:\n%s", err, out) } if runs := firstPassRuns(out); len(runs) != 2 { t.Errorf("got %d first-pass go test invocations, want 2 (one per file):\n%s\noutput:\n%s", len(runs), strings.Join(runs, "\n"), out) } wantRetry := fileB + " --run ^(TestFlaky)$" if !bytes.Contains(out, []byte(wantRetry)) { t.Errorf("output lacks retry of %q:\n%s", wantRetry, out) } if bytes.Contains(out, []byte("permanent test failures JSON:")) { t.Errorf("unexpected permanent failures:\n%s", out) }}