testing: write output to buffer when fuzzing

Fixes #48709

Change-Id: Ia6376a2f792946498d6565a53605b3e6c985ea7c
Reviewed-on: https://go-review.googlesource.com/c/go/+/355909
Trust: Katie Hockman <katie@golang.org>
Run-TryBot: Katie Hockman <katie@golang.org>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Jay Conrod <jayconrod@google.com>
Reviewed-by: Roland Shoemaker <roland@golang.org>
Reviewed-by: Bryan C. Mills <bcmills@google.com>

src/cmd/go/testdata/script/test_fuzz_minimize.txt

@@ -31,7 +31,7 @@ stdout FAIL
 ! go test -fuzz=FuzzMinimizeZeroLimitSet -run=FuzzMinimizeZeroLimitSet -fuzztime=10000x -fuzzminimizetime=0x .
 ! stdout '^ok'
 ! stdout 'minimizing'
-stdout 'there was an Error'
+stdout -count=1 'there was an Error'
 stdout FAIL
 
 # Test that minimization is working for recoverable errors.
@@ -49,11 +49,27 @@ go run ./check_testdata FuzzMinimizerRecoverable 50
 ! go test -run=FuzzMinimizerRecoverable .
 rm testdata
 
+# Test that minimization is working for recoverable errors. Run it with -v this
+# time to ensure the command line output still looks right.
+! go test -v -fuzz=FuzzMinimizerRecoverable -run=FuzzMinimizerRecoverable -fuzztime=10000x .
+! stdout '^ok'
+stdout 'got the minimum size!'
+# The error message that was printed should be for the one written to testdata.
+stdout 'contains a non-zero byte of length 50'
+stdout FAIL
+
+# Check that the bytes written to testdata are of length 50 (the minimum size)
+go run ./check_testdata FuzzMinimizerRecoverable 50
+
+# Test that re-running the minimized value causes a crash.
+! go test -run=FuzzMinimizerRecoverable .
+rm testdata
+
 # Test that minimization doesn't run for non-recoverable errors.
 ! go test -fuzz=FuzzMinimizerNonrecoverable -run=FuzzMinimizerNonrecoverable -fuzztime=10000x .
 ! stdout '^ok'
 ! stdout 'minimizing'
-stdout 'fuzzing process terminated unexpectedly: exit status 99'
+stdout -count=1 'fuzzing process terminated unexpectedly: exit status 99'
 stdout FAIL
 
 # Check that re-running the value causes a crash.

src/cmd/go/testdata/script/test_fuzz_minimize_interesting.txt

@@ -29,7 +29,7 @@ env GOCACHE=$WORK/gocache
 ! exec ./fuzz.test$GOEXE -test.fuzzcachedir=$GOCACHE/fuzz -test.fuzz=FuzzMinimizerCrashInMinimization -test.fuzztime=10000x -test.parallel=1
 ! stdout '^ok'
 stdout 'got the minimum size!'
-stdout 'flaky failure'
+stdout -count=1 'flaky failure'
 stdout FAIL
 
 # Make sure the crash that was written will fail when run with go test

src/testing/fuzz.go

@@ -5,6 +5,7 @@
 package testing
 
 import (
+	"bytes"
 	"errors"
 	"flag"
 	"fmt"
@@ -367,14 +368,14 @@ func (f *F) Fuzz(ff interface{}) {
 	// run calls fn on a given input, as a subtest with its own T.
 	// run is analogous to T.Run. The test filtering and cleanup works similarly.
 	// fn is called in its own goroutine.
-	run := func(e corpusEntry) error {
+	run := func(captureOut io.Writer, e corpusEntry) (ok bool) {
 		if e.Values == nil {
 			// The corpusEntry must have non-nil Values in order to run the
 			// test. If Values is nil, it is a bug in our code.
 			panic(fmt.Sprintf("corpus file %q was not unmarshaled", e.Path))
 		}
 		if shouldFailFast() {
-			return nil
+			return true
 		}
 		testName := f.name
 		if e.Path != "" {
@@ -405,6 +406,10 @@ func (f *F) Fuzz(ff interface{}) {
 			},
 			context: f.testContext,
 		}
+		if captureOut != nil {
+			// t.parent aliases f.common.
+			t.parent.w = captureOut
+		}
 		t.w = indenter{&t.common}
 		if t.chatty != nil {
 			// TODO(#48132): adjust this to work with test2json.
@@ -426,10 +431,7 @@ func (f *F) Fuzz(ff interface{}) {
 		})
 		<-t.signal
 		f.inFuzzFn = false
-		if t.Failed() {
-			return errors.New(string(f.output))
-		}
-		return nil
+		return !t.Failed()
 	}
 
 	switch f.fuzzContext.mode {
@@ -466,7 +468,17 @@ func (f *F) Fuzz(ff interface{}) {
 	case fuzzWorker:
 		// Fuzzing is enabled, and this is a worker process. Follow instructions
 		// from the coordinator.
-		if err := f.fuzzContext.deps.RunFuzzWorker(run); err != nil {
+		if err := f.fuzzContext.deps.RunFuzzWorker(func(e corpusEntry) error {
+			// Don't write to f.w (which points to Stdout) if running from a
+			// fuzz worker. This would become very verbose, particularly during
+			// minimization. Return the error instead, and let the caller deal
+			// with the output.
+			var buf bytes.Buffer
+			if ok := run(&buf, e); !ok {
+				return errors.New(buf.String())
+			}
+			return nil
+		}); err != nil {
 			// Internal errors are marked with f.Fail; user code may call this too, before F.Fuzz.
 			// The worker will exit with fuzzWorkerExitCode, indicating this is a failure
 			// (and 'go test' should exit non-zero) but a crasher should not be recorded.
@@ -479,7 +491,7 @@ func (f *F) Fuzz(ff interface{}) {
 		for _, e := range f.corpus {
 			name := fmt.Sprintf("%s/%s", f.name, filepath.Base(e.Path))
 			if _, ok, _ := f.testContext.match.fullName(nil, name); ok {
-				run(e)
+				run(f.w, e)
 			}
 		}
 	}