cmd/go, testing: add go test -skip flag

For proposal #41583, add a new 'go test -skip' flag to make it easy
to disable specific tests, benchmarks, examples, or fuzz targets.

Fixes #41583.

Change-Id: Id12a6575f505dafdce4a149aedc454a002e93afa
Reviewed-on: https://go-review.googlesource.com/c/go/+/421439
TryBot-Result: Gopher Robot <gobot@golang.org>
Run-TryBot: Russ Cox <rsc@golang.org>
Reviewed-by: Bryan Mills <bcmills@google.com>

src/cmd/go/alldocs.go

@@ -2983,6 +2983,7 @@
 //	    run too, so that -run=X/Y matches and runs and reports the result
 //	    of all tests matching X, even those without sub-tests matching Y,
 //	    because it must run them to look for those sub-tests.
+//	    See also -skip.
 //
 //	-short
 //	    Tell long-running tests to shorten their run time.
@@ -2997,6 +2998,14 @@
 //	    integer N, then N will be used as the seed value. In both cases,
 //	    the seed will be reported for reproducibility.
 //
+//	-skip regexp
+//	    Run only those tests, examples, fuzz tests, and benchmarks that
+//	    do not match the regular expression. Like for -run and -bench,
+//	    for tests and benchmarks, the regular expression is split by unbracketed
+//	    slash (/) characters into a sequence of regular expressions, and each
+//	    part of a test's identifier must match the corresponding element in
+//	    the sequence, if any.
+//
 //	-timeout d
 //	    If a test binary runs longer than duration d, panic.
 //	    If d is 0, the timeout is disabled.

src/cmd/go/internal/test/flagdefs.go

@@ -32,6 +32,7 @@ var passFlagToTest = map[string]bool{
 	"run":                  true,
 	"short":                true,
 	"shuffle":              true,
+	"skip":                 true,
 	"timeout":              true,
 	"trace":                true,
 	"v":                    true,

src/cmd/go/internal/test/test.go

@@ -306,6 +306,7 @@ control the execution of any test:
 	    run too, so that -run=X/Y matches and runs and reports the result
 	    of all tests matching X, even those without sub-tests matching Y,
 	    because it must run them to look for those sub-tests.
+	    See also -skip.
 
 	-short
 	    Tell long-running tests to shorten their run time.
@@ -320,6 +321,14 @@ control the execution of any test:
 	    integer N, then N will be used as the seed value. In both cases,
 	    the seed will be reported for reproducibility.
 
+	-skip regexp
+	    Run only those tests, examples, fuzz tests, and benchmarks that
+	    do not match the regular expression. Like for -run and -bench,
+	    for tests and benchmarks, the regular expression is split by unbracketed
+	    slash (/) characters into a sequence of regular expressions, and each
+	    part of a test's identifier must match the corresponding element in
+	    the sequence, if any.
+
 	-timeout d
 	    If a test binary runs longer than duration d, panic.
 	    If d is 0, the timeout is disabled.

src/cmd/go/internal/test/testflag.go

@@ -66,6 +66,7 @@ func init() {
 	cf.Int("parallel", 0, "")
 	cf.String("run", "", "")
 	cf.Bool("short", false, "")
+	cf.String("skip", "", "")
 	cf.DurationVar(&testTimeout, "timeout", 10*time.Minute, "")
 	cf.String("fuzztime", "", "")
 	cf.String("fuzzminimizetime", "", "")

src/cmd/go/testdata/script/test_skip.txt

@@ -0,0 +1,34 @@
+go test -v -run Test -skip T skip_test.go
+! stdout RUN
+stdout '^ok.*\[no tests to run\]'
+
+go test -v -skip T skip_test.go
+! stdout RUN
+
+go test -v -skip 1 skip_test.go
+! stdout Test1
+stdout RUN.*Test2
+stdout RUN.*Test2/3
+
+go test -v -skip 2/3 skip_test.go
+stdout RUN.*Test1
+stdout RUN.*Test2
+! stdout Test2/3
+
+go test -v -skip 2/4 skip_test.go
+stdout RUN.*Test1
+stdout RUN.*Test2
+stdout RUN.*Test2/3
+
+
+-- skip_test.go --
+package skip_test
+
+import "testing"
+
+func Test1(t *testing.T) {
+}
+
+func Test2(t *testing.T) {
+	t.Run("3", func(t *testing.T) {})
+}

src/testing/benchmark.go

@@ -536,7 +536,7 @@ func runBenchmarks(importPath string, matchString func(pat, str string) (bool, e
 		}
 	}
 	ctx := &benchContext{
-		match:  newMatcher(matchString, *matchBenchmarks, "-test.bench"),
+		match:  newMatcher(matchString, *matchBenchmarks, "-test.bench", *skip),
 		extLen: len(benchmarkName("", maxprocs)),
 	}
 	var bs []InternalBenchmark

src/testing/fuzz.go

@@ -471,12 +471,12 @@ func runFuzzTests(deps testDeps, fuzzTests []InternalFuzzTarget, deadline time.T
 	if len(fuzzTests) == 0 || *isFuzzWorker {
 		return ran, ok
 	}
-	m := newMatcher(deps.MatchString, *match, "-test.run")
+	m := newMatcher(deps.MatchString, *match, "-test.run", *skip)
 	tctx := newTestContext(*parallel, m)
 	tctx.deadline = deadline
 	var mFuzz *matcher
 	if *matchFuzz != "" {
-		mFuzz = newMatcher(deps.MatchString, *matchFuzz, "-test.fuzz")
+		mFuzz = newMatcher(deps.MatchString, *matchFuzz, "-test.fuzz", *skip)
 	}
 	fctx := &fuzzContext{deps: deps, mode: seedCorpusOnly}
 	root := common{w: os.Stdout} // gather output in one place
@@ -532,7 +532,7 @@ func runFuzzing(deps testDeps, fuzzTests []InternalFuzzTarget) (ok bool) {
 	if len(fuzzTests) == 0 || *matchFuzz == "" {
 		return true
 	}
-	m := newMatcher(deps.MatchString, *matchFuzz, "-test.fuzz")
+	m := newMatcher(deps.MatchString, *matchFuzz, "-test.fuzz", *skip)
 	tctx := newTestContext(1, m)
 	tctx.isFuzzing = true
 	fctx := &fuzzContext{

src/testing/helper_test.go

@@ -11,7 +11,7 @@ import (
 
 func TestTBHelper(t *T) {
 	var buf strings.Builder
-	ctx := newTestContext(1, newMatcher(regexp.MatchString, "", ""))
+	ctx := newTestContext(1, allMatcher())
 	t1 := &T{
 		common: common{
 			signal: make(chan bool),
@@ -55,7 +55,7 @@ helperfuncs_test.go:67: 10
 
 func TestTBHelperParallel(t *T) {
 	var buf strings.Builder
-	ctx := newTestContext(1, newMatcher(regexp.MatchString, "", ""))
+	ctx := newTestContext(1, newMatcher(regexp.MatchString, "", "", ""))
 	t1 := &T{
 		common: common{
 			signal: make(chan bool),
@@ -81,7 +81,7 @@ func (nw *noopWriter) Write(b []byte) (int, error) { return len(b), nil }
 
 func BenchmarkTBHelper(b *B) {
 	w := noopWriter(0)
-	ctx := newTestContext(1, newMatcher(regexp.MatchString, "", ""))
+	ctx := newTestContext(1, allMatcher())
 	t1 := &T{
 		common: common{
 			signal: make(chan bool),

src/testing/match.go

@@ -15,6 +15,7 @@ import (
 // matcher sanitizes, uniques, and filters names of subtests and subbenchmarks.
 type matcher struct {
 	filter    filterMatch
+	skip      filterMatch
 	matchFunc func(pat, str string) (bool, error)
 
 	mu sync.Mutex
@@ -47,17 +48,33 @@ type alternationMatch []filterMatch
 // eliminate this Mutex.
 var matchMutex sync.Mutex
 
-func newMatcher(matchString func(pat, str string) (bool, error), patterns, name string) *matcher {
-	var impl filterMatch
-	if patterns != "" {
-		impl = splitRegexp(patterns)
-		if err := impl.verify(name, matchString); err != nil {
+func allMatcher() *matcher {
+	return newMatcher(nil, "", "", "")
+}
+
+func newMatcher(matchString func(pat, str string) (bool, error), patterns, name, skips string) *matcher {
+	var filter, skip filterMatch
+	if patterns == "" {
+		filter = simpleMatch{} // always partial true
+	} else {
+		filter = splitRegexp(patterns)
+		if err := filter.verify(name, matchString); err != nil {
 			fmt.Fprintf(os.Stderr, "testing: invalid regexp for %s\n", err)
 			os.Exit(1)
 		}
 	}
+	if skips == "" {
+		skip = alternationMatch{} // always false
+	} else {
+		skip = splitRegexp(skips)
+		if err := skip.verify("-test.skip", matchString); err != nil {
+			fmt.Fprintf(os.Stderr, "testing: invalid regexp for %v\n", err)
+			os.Exit(1)
+		}
+	}
 	return &matcher{
-		filter:    impl,
+		filter:    filter,
+		skip:      skip,
 		matchFunc: matchString,
 		subNames:  map[string]int32{},
 	}
@@ -76,14 +93,23 @@ func (m *matcher) fullName(c *common, subname string) (name string, ok, partial
 	matchMutex.Lock()
 	defer matchMutex.Unlock()
 
-	if m.filter == nil {
-		return name, true, false
+	// We check the full array of paths each time to allow for the case that a pattern contains a '/'.
+	elem := strings.Split(name, "/")
+
+	// filter must match.
+	// accept partial match that may produce full match later.
+	ok, partial = m.filter.matches(elem, m.matchFunc)
+	if !ok {
+		return name, false, false
+	}
+
+	// skip must not match.
+	// ignore partial match so we can get to more precise match later.
+	skip, partialSkip := m.skip.matches(elem, m.matchFunc)
+	if skip && !partialSkip {
+		return name, false, false
 	}
 
-	// We check the full array of paths each time to allow for the case that
-	// a pattern contains a '/'.
-	elem := strings.Split(name, "/")
-	ok, partial = m.filter.matches(elem, m.matchFunc)
 	return name, ok, partial
 }
 

src/testing/match_test.go

@@ -12,6 +12,10 @@ import (
 	"unicode"
 )
 
+func init() {
+	testingTesting = true
+}
+
 // Verify that our IsSpace agrees with unicode.IsSpace.
 func TestIsSpace(t *T) {
 	n := 0
@@ -89,54 +93,75 @@ func TestSplitRegexp(t *T) {
 func TestMatcher(t *T) {
 	testCases := []struct {
 		pattern     string
+		skip        string
 		parent, sub string
 		ok          bool
 		partial     bool
 	}{
 		// Behavior without subtests.
-		{"", "", "TestFoo", true, false},
-		{"TestFoo", "", "TestFoo", true, false},
-		{"TestFoo/", "", "TestFoo", true, true},
-		{"TestFoo/bar/baz", "", "TestFoo", true, true},
-		{"TestFoo", "", "TestBar", false, false},
-		{"TestFoo/", "", "TestBar", false, false},
-		{"TestFoo/bar/baz", "", "TestBar/bar/baz", false, false},
+		{"", "", "", "TestFoo", true, false},
+		{"TestFoo", "", "", "TestFoo", true, false},
+		{"TestFoo/", "", "", "TestFoo", true, true},
+		{"TestFoo/bar/baz", "", "", "TestFoo", true, true},
+		{"TestFoo", "", "", "TestBar", false, false},
+		{"TestFoo/", "", "", "TestBar", false, false},
+		{"TestFoo/bar/baz", "", "", "TestBar/bar/baz", false, false},
+		{"", "TestBar", "", "TestFoo", true, false},
+		{"", "TestBar", "", "TestBar", false, false},
+
+		// Skipping a non-existent test doesn't change anything.
+		{"", "TestFoo/skipped", "", "TestFoo", true, false},
+		{"TestFoo", "TestFoo/skipped", "", "TestFoo", true, false},
+		{"TestFoo/", "TestFoo/skipped", "", "TestFoo", true, true},
+		{"TestFoo/bar/baz", "TestFoo/skipped", "", "TestFoo", true, true},
+		{"TestFoo", "TestFoo/skipped", "", "TestBar", false, false},
+		{"TestFoo/", "TestFoo/skipped", "", "TestBar", false, false},
+		{"TestFoo/bar/baz", "TestFoo/skipped", "", "TestBar/bar/baz", false, false},
 
 		// with subtests
-		{"", "TestFoo", "x", true, false},
-		{"TestFoo", "TestFoo", "x", true, false},
-		{"TestFoo/", "TestFoo", "x", true, false},
-		{"TestFoo/bar/baz", "TestFoo", "bar", true, true},
+		{"", "", "TestFoo", "x", true, false},
+		{"TestFoo", "", "TestFoo", "x", true, false},
+		{"TestFoo/", "", "TestFoo", "x", true, false},
+		{"TestFoo/bar/baz", "", "TestFoo", "bar", true, true},
+
+		{"", "TestFoo/skipped", "TestFoo", "x", true, false},
+		{"TestFoo", "TestFoo/skipped", "TestFoo", "x", true, false},
+		{"TestFoo", "TestFoo/skipped", "TestFoo", "skipped", false, false},
+		{"TestFoo/", "TestFoo/skipped", "TestFoo", "x", true, false},
+		{"TestFoo/bar/baz", "TestFoo/skipped", "TestFoo", "bar", true, true},
+
 		// Subtest with a '/' in its name still allows for copy and pasted names
 		// to match.
-		{"TestFoo/bar/baz", "TestFoo", "bar/baz", true, false},
-		{"TestFoo/bar/baz", "TestFoo/bar", "baz", true, false},
-		{"TestFoo/bar/baz", "TestFoo", "x", false, false},
-		{"TestFoo", "TestBar", "x", false, false},
-		{"TestFoo/", "TestBar", "x", false, false},
-		{"TestFoo/bar/baz", "TestBar", "x/bar/baz", false, false},
+		{"TestFoo/bar/baz", "", "TestFoo", "bar/baz", true, false},
+		{"TestFoo/bar/baz", "TestFoo/bar/baz", "TestFoo", "bar/baz", false, false},
+		{"TestFoo/bar/baz", "TestFoo/bar/baz/skip", "TestFoo", "bar/baz", true, false},
+		{"TestFoo/bar/baz", "", "TestFoo/bar", "baz", true, false},
+		{"TestFoo/bar/baz", "", "TestFoo", "x", false, false},
+		{"TestFoo", "", "TestBar", "x", false, false},
+		{"TestFoo/", "", "TestBar", "x", false, false},
+		{"TestFoo/bar/baz", "", "TestBar", "x/bar/baz", false, false},
 
-		{"A/B|C/D", "TestA", "B", true, false},
-		{"A/B|C/D", "TestC", "D", true, false},
-		{"A/B|C/D", "TestA", "C", false, false},
+		{"A/B|C/D", "", "TestA", "B", true, false},
+		{"A/B|C/D", "", "TestC", "D", true, false},
+		{"A/B|C/D", "", "TestA", "C", false, false},
 
 		// subtests only
-		{"", "TestFoo", "x", true, false},
-		{"/", "TestFoo", "x", true, false},
-		{"./", "TestFoo", "x", true, false},
-		{"./.", "TestFoo", "x", true, false},
-		{"/bar/baz", "TestFoo", "bar", true, true},
-		{"/bar/baz", "TestFoo", "bar/baz", true, false},
-		{"//baz", "TestFoo", "bar/baz", true, false},
-		{"//", "TestFoo", "bar/baz", true, false},
-		{"/bar/baz", "TestFoo/bar", "baz", true, false},
-		{"//foo", "TestFoo", "bar/baz", false, false},
-		{"/bar/baz", "TestFoo", "x", false, false},
-		{"/bar/baz", "TestBar", "x/bar/baz", false, false},
+		{"", "", "TestFoo", "x", true, false},
+		{"/", "", "TestFoo", "x", true, false},
+		{"./", "", "TestFoo", "x", true, false},
+		{"./.", "", "TestFoo", "x", true, false},
+		{"/bar/baz", "", "TestFoo", "bar", true, true},
+		{"/bar/baz", "", "TestFoo", "bar/baz", true, false},
+		{"//baz", "", "TestFoo", "bar/baz", true, false},
+		{"//", "", "TestFoo", "bar/baz", true, false},
+		{"/bar/baz", "", "TestFoo/bar", "baz", true, false},
+		{"//foo", "", "TestFoo", "bar/baz", false, false},
+		{"/bar/baz", "", "TestFoo", "x", false, false},
+		{"/bar/baz", "", "TestBar", "x/bar/baz", false, false},
 	}
 
 	for _, tc := range testCases {
-		m := newMatcher(regexp.MatchString, tc.pattern, "-test.run")
+		m := newMatcher(regexp.MatchString, tc.pattern, "-test.run", tc.skip)
 
 		parent := &common{name: tc.parent}
 		if tc.parent != "" {
@@ -184,7 +209,7 @@ var namingTestCases = []struct{ name, want string }{
 }
 
 func TestNaming(t *T) {
-	m := newMatcher(regexp.MatchString, "", "")
+	m := newMatcher(regexp.MatchString, "", "", "")
 	parent := &common{name: "x", level: 1} // top-level test.
 
 	for i, tc := range namingTestCases {
@@ -202,7 +227,7 @@ func FuzzNaming(f *F) {
 	var m *matcher
 	var seen map[string]string
 	reset := func() {
-		m = newMatcher(regexp.MatchString, "", "")
+		m = allMatcher()
 		seen = make(map[string]string)
 	}
 	reset()

src/testing/sub_test.go

@@ -476,7 +476,7 @@ func TestTRun(t *T) {
 	}}
 	for _, tc := range testCases {
 		t.Run(tc.desc, func(t *T) {
-			ctx := newTestContext(tc.maxPar, newMatcher(regexp.MatchString, "", ""))
+			ctx := newTestContext(tc.maxPar, allMatcher())
 			buf := &strings.Builder{}
 			root := &T{
 				common: common{
@@ -775,7 +775,7 @@ func TestRacyOutput(t *T) {
 	var wg sync.WaitGroup
 	root := &T{
 		common:  common{w: &funcWriter{raceDetector}},
-		context: newTestContext(1, newMatcher(regexp.MatchString, "", "")),
+		context: newTestContext(1, allMatcher()),
 	}
 	root.chatty = newChattyPrinter(root.w)
 	root.Run("", func(t *T) {
@@ -798,7 +798,7 @@ func TestRacyOutput(t *T) {
 
 // The late log message did not include the test name.  Issue 29388.
 func TestLogAfterComplete(t *T) {
-	ctx := newTestContext(1, newMatcher(regexp.MatchString, "", ""))
+	ctx := newTestContext(1, allMatcher())
 	var buf bytes.Buffer
 	t1 := &T{
 		common: common{

src/testing/testing.go

@@ -422,6 +422,7 @@ func Init() {
 	coverProfile = flag.String("test.coverprofile", "", "write a coverage profile to `file`")
 	matchList = flag.String("test.list", "", "list tests, examples, and benchmarks matching `regexp` then exit")
 	match = flag.String("test.run", "", "run only tests and examples matching `regexp`")
+	skip = flag.String("test.skip", "", "do not list or run tests matching `regexp`")
 	memProfile = flag.String("test.memprofile", "", "write an allocation profile to `file`")
 	memProfileRate = flag.Int("test.memprofilerate", 0, "set memory allocation profiling `rate` (see runtime.MemProfileRate)")
 	cpuProfile = flag.String("test.cpuprofile", "", "write a cpu profile to `file`")
@@ -451,6 +452,7 @@ var (
 	coverProfile         *string
 	matchList            *string
 	match                *string
+	skip                 *string
 	memProfile           *string
 	memProfileRate       *int
 	cpuProfile           *string
@@ -1690,6 +1692,8 @@ func MainStart(deps testDeps, tests []InternalTest, benchmarks []InternalBenchma
 	}
 }
 
+var testingTesting bool
+
 // Run runs the tests. It returns an exit code to pass to os.Exit.
 func (m *M) Run() (code int) {
 	defer func() {
@@ -1720,7 +1724,7 @@ func (m *M) Run() (code int) {
 		return
 	}
 
-	if len(*matchList) != 0 {
+	if *matchList != "" {
 		listTests(m.deps.MatchString, m.tests, m.benchmarks, m.fuzzTargets, m.examples)
 		m.exitCode = 0
 		return
@@ -1762,6 +1766,15 @@ func (m *M) Run() (code int) {
 		m.stopAlarm()
 		if !testRan && !exampleRan && !fuzzTargetsRan && *matchBenchmarks == "" && *matchFuzz == "" {
 			fmt.Fprintln(os.Stderr, "testing: warning: no tests to run")
+			if testingTesting {
+				// If this happens during testing of package testing it could be that
+				// package testing's own logic for when to run a test is broken,
+				// in which case every test will run nothing and succeed,
+				// with no obvious way to detect this problem (since no tests are running).
+				// So make 'no tests to run' a hard failure when testing package testing itself.
+				fmt.Println("FAIL: package testing must run tests")
+				testOk = false
+			}
 		}
 		if !testOk || !exampleOk || !fuzzTargetsOk || !runBenchmarks(m.deps.ImportPath(), m.deps.MatchString, m.benchmarks) || race.Errors() > 0 {
 			fmt.Println("FAIL")
@@ -1861,7 +1874,7 @@ func runTests(matchString func(pat, str string) (bool, error), tests []InternalT
 				// to keep trying.
 				break
 			}
-			ctx := newTestContext(*parallel, newMatcher(matchString, *match, "-test.run"))
+			ctx := newTestContext(*parallel, newMatcher(matchString, *match, "-test.run", *skip))
 			ctx.deadline = deadline
 			t := &T{
 				common: common{