mirror of
https://github.com/gravitational/teleport
synced 2024-10-19 16:53:57 +00:00
Include package-level failures in formatted test output (#8698)
In some cases, it's possible for a package to be marked as a test failure even if no tests inside it have failed. The motivating example for this change is a timeout: a test overshooting the allotted timeout is considered by go test to be a package-level failure, even if no tests inside the package are considered failures. This led to cases where the user would see an "All tests passed" message from the go test filter, but still mysteriously fail the make step. To address this, the test renderer now: * treats package-level pass/fail/skip events as first-class citizens and includes them in its event count, * tracks the cached test output at both a package and individual test level, and * displays the whole package output if a package is marked as failed, but only if there is no obvious failed test top account for the package-level failure. This patch also removes the json files created by the unit tests, as they are not yet needed for anything.
This commit is contained in:
parent
a731e3c9ff
commit
eca9603376
5
Makefile
5
Makefile
|
@ -281,6 +281,7 @@ endif
|
|||
.PHONY: clean
|
||||
clean:
|
||||
@echo "---> Cleaning up OSS build artifacts."
|
||||
rm -rf build.assets/build
|
||||
rm -rf $(BUILDDIR)
|
||||
rm -rf $(ER_BPF_BUILDDIR)
|
||||
rm -rf $(RS_BPF_BUILDDIR)
|
||||
|
@ -441,10 +442,8 @@ test-go: PACKAGES := $(shell go list ./... | grep -v integration)
|
|||
test-go: CHAOS_FOLDERS := $(shell find . -type f -name '*chaos*.go' -not -path '*/vendor/*' | xargs dirname | uniq)
|
||||
test-go: $(VERSRC)
|
||||
$(CGOFLAG) go test -p 4 -cover -json -tags "$(PAM_TAG) $(FIPS_TAG) $(BPF_TAG) $(ROLETESTER_TAG) $(DESKTOP_ACCESS_BETA_TAG)" $(PACKAGES) $(FLAGS) $(ADDFLAGS) \
|
||||
| tee tests-unit.json \
|
||||
| go run build.assets/render-tests/main.go
|
||||
$(CGOFLAG) go test -p 4 -cover -json -tags "$(PAM_TAG) $(FIPS_TAG) $(BPF_TAG) $(ROLETESTER_TAG) $(DESKTOP_ACCESS_BETA_TAG)" -test.run=TestChaos $(CHAOS_FOLDERS) -cover \
|
||||
| tee -a tests-unit.json \
|
||||
$(CGOFLAG) go test -p 4 -cover -json -tags "$(PAM_TAG) $(FIPS_TAG) $(BPF_TAG) $(ROLETESTER_TAG) $(DESKTOP_ACCESS_BETA_TAG)" -test.run=TestChaos $(CHAOS_FOLDERS) \
|
||||
| go run build.assets/render-tests/main.go
|
||||
|
||||
#
|
||||
|
|
|
@ -81,11 +81,82 @@ func readInput(input io.Reader, ch chan<- TestEvent) {
|
|||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
type packageOutput struct {
|
||||
output []string
|
||||
subtestOutput map[string][]string
|
||||
failedSubtests map[string][]string
|
||||
}
|
||||
|
||||
testOutput := make(map[string][]string)
|
||||
failedTests := make(map[string][]string)
|
||||
actionCounts := make(map[string]int)
|
||||
func (pkg *packageOutput) FailedTests() []string {
|
||||
result := []string{}
|
||||
for testName := range pkg.failedSubtests {
|
||||
result = append(result, testName)
|
||||
}
|
||||
sort.Strings(result)
|
||||
return result
|
||||
}
|
||||
|
||||
type outputMap struct {
|
||||
pkgs map[string]*packageOutput
|
||||
actionCounts map[string]int
|
||||
}
|
||||
|
||||
func newOutputMap() *outputMap {
|
||||
return &outputMap{
|
||||
pkgs: make(map[string]*packageOutput),
|
||||
actionCounts: make(map[string]int),
|
||||
}
|
||||
}
|
||||
|
||||
func (m *outputMap) record(event TestEvent) {
|
||||
m.actionCounts[event.Action]++
|
||||
|
||||
var pkgOutput *packageOutput
|
||||
var exists bool
|
||||
if pkgOutput, exists = m.pkgs[event.Package]; !exists {
|
||||
pkgOutput = &packageOutput{
|
||||
subtestOutput: make(map[string][]string),
|
||||
failedSubtests: make(map[string][]string),
|
||||
}
|
||||
m.pkgs[event.Package] = pkgOutput
|
||||
}
|
||||
|
||||
switch event.Action {
|
||||
case actionOutput:
|
||||
pkgOutput.output = append(pkgOutput.output, event.Output)
|
||||
if event.Test != "" {
|
||||
pkgOutput.subtestOutput[event.Test] = append(pkgOutput.subtestOutput[event.Test], event.Output)
|
||||
}
|
||||
|
||||
case actionFail:
|
||||
// If this is a single test result
|
||||
if event.Test != "" {
|
||||
pkgOutput.failedSubtests[event.Test] = pkgOutput.subtestOutput[event.Test]
|
||||
} else {
|
||||
// If this is a package result, we only want to preserve the package output if
|
||||
// there are no failed subtests
|
||||
if len(pkgOutput.failedSubtests) > 0 {
|
||||
pkgOutput.output = nil
|
||||
}
|
||||
}
|
||||
fallthrough
|
||||
|
||||
case actionPass, actionSkip:
|
||||
delete(pkgOutput.subtestOutput, event.Test)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *outputMap) getPkg(pkgName string) *packageOutput {
|
||||
return m.pkgs[pkgName]
|
||||
}
|
||||
|
||||
func (m *outputMap) deletePkg(pkgName string) {
|
||||
delete(m.pkgs, pkgName)
|
||||
}
|
||||
|
||||
func main() {
|
||||
testOutput := newOutputMap()
|
||||
failedPackages := make(map[string]*packageOutput)
|
||||
coverage := make(map[string]float64)
|
||||
|
||||
events := make(chan TestEvent)
|
||||
|
@ -109,20 +180,26 @@ func main() {
|
|||
|
||||
testName := event.FullName()
|
||||
|
||||
switch event.Action {
|
||||
case actionOutput:
|
||||
if matches := covPattern.FindStringSubmatch(event.Output); len(matches) != 0 {
|
||||
value, err := strconv.ParseFloat(matches[1], 64)
|
||||
if err != nil {
|
||||
panic("Malformed coverage value: " + err.Error())
|
||||
}
|
||||
coverage[testName] = value
|
||||
}
|
||||
testOutput[testName] = append(testOutput[testName], event.Output)
|
||||
testOutput.record(event)
|
||||
|
||||
case actionPass, actionFail, actionSkip:
|
||||
// If this is a whole-package summary result
|
||||
if event.Test == "" {
|
||||
// if this is whole-package summary result
|
||||
if event.Test == "" {
|
||||
switch event.Action {
|
||||
case actionOutput:
|
||||
if matches := covPattern.FindStringSubmatch(event.Output); len(matches) != 0 {
|
||||
value, err := strconv.ParseFloat(matches[1], 64)
|
||||
if err != nil {
|
||||
panic("Malformed coverage value: " + err.Error())
|
||||
}
|
||||
coverage[testName] = value
|
||||
}
|
||||
|
||||
case actionFail:
|
||||
// cache the failed test output
|
||||
failedPackages[event.Package] = testOutput.getPkg(event.Package)
|
||||
fallthrough
|
||||
|
||||
case actionPass, actionSkip:
|
||||
// extract and format coverage value
|
||||
covText := "------"
|
||||
if covValue, ok := coverage[testName]; ok {
|
||||
|
@ -131,20 +208,10 @@ func main() {
|
|||
|
||||
// only display package results as progress messages
|
||||
fmt.Printf("%s %s: %s\n", covText, event.Action, event.Package)
|
||||
} else {
|
||||
// packages results don't count towards our test count.
|
||||
actionCounts[event.Action]++
|
||||
|
||||
// we want to preserve the log of our failed tests for
|
||||
// later examination
|
||||
if event.Action == actionFail {
|
||||
failedTests[testName] = testOutput[testName]
|
||||
}
|
||||
// Don't need this no more
|
||||
testOutput.deletePkg(event.Package)
|
||||
}
|
||||
// we don't need the test output any more, it's either in the
|
||||
// errors collection, or the test passed and we don't need to
|
||||
// display it.
|
||||
delete(testOutput, testName)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -152,33 +219,58 @@ func main() {
|
|||
fmt.Println(separator)
|
||||
|
||||
fmt.Printf("%d tests passed. %d failed, %d skipped\n",
|
||||
actionCounts[actionPass], actionCounts[actionFail], actionCounts[actionSkip])
|
||||
testOutput.actionCounts[actionPass], testOutput.actionCounts[actionFail], testOutput.actionCounts[actionSkip])
|
||||
|
||||
fmt.Println(separator)
|
||||
|
||||
if len(failedTests) == 0 {
|
||||
if len(failedPackages) == 0 {
|
||||
fmt.Println("All tests pass. Yay!")
|
||||
return
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
names := make([]string, 0, len(failedTests))
|
||||
for k := range failedTests {
|
||||
// Generate a sorted list of package names, so that we present the
|
||||
// packages that fail in a repeatable order.
|
||||
names := make([]string, 0, len(failedPackages))
|
||||
for k := range failedPackages {
|
||||
names = append(names, k)
|
||||
}
|
||||
sort.Strings(names)
|
||||
|
||||
for _, testName := range names {
|
||||
fmt.Printf("FAIL: %s\n", testName)
|
||||
}
|
||||
// Print a summary list of the failed tests and packages.
|
||||
for _, pkgName := range names {
|
||||
pkg := failedPackages[pkgName]
|
||||
|
||||
for _, testName := range names {
|
||||
fmt.Println(separator)
|
||||
fmt.Printf("TEST %s OUTPUT\n", testName)
|
||||
fmt.Println(separator)
|
||||
for _, l := range failedTests[testName] {
|
||||
fmt.Print(l)
|
||||
fmt.Printf("FAIL: %s\n", pkgName)
|
||||
|
||||
for testName := range pkg.failedSubtests {
|
||||
fmt.Printf("FAIL: %s.%s\n", pkgName, testName)
|
||||
}
|
||||
}
|
||||
|
||||
os.Exit(1)
|
||||
fmt.Println(separator)
|
||||
|
||||
// Print the output of each failed package or test. Note that we only print
|
||||
// the package output if there is no identifiable test that caused the
|
||||
// failure, as it will probably swamp the individual test output.
|
||||
for _, pkgName := range names {
|
||||
pkg := failedPackages[pkgName]
|
||||
|
||||
if len(pkg.failedSubtests) == 0 {
|
||||
fmt.Printf("OUTPUT %s\n", pkgName)
|
||||
fmt.Println(separator)
|
||||
for _, l := range pkg.output {
|
||||
fmt.Print(l)
|
||||
}
|
||||
fmt.Println(separator)
|
||||
} else {
|
||||
for _, testName := range pkg.FailedTests() {
|
||||
fmt.Printf("OUTPUT %s.%s\n", pkgName, testName)
|
||||
fmt.Println(separator)
|
||||
for _, l := range pkg.failedSubtests[testName] {
|
||||
fmt.Print(l)
|
||||
}
|
||||
fmt.Println(separator)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue