self-hosted/teleport
self-hosted/teleport
1
0
Fork 0
mirror of https://github.com/gravitational/teleport synced 2024-10-19 16:53:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Include package-level failures in formatted test output (#8698)

Browse source 
In some cases, it's possible for a package to be marked as a test
failure even if no tests inside it have failed. The motivating example
for this change is a timeout: a test overshooting the allotted timeout
is considered by go test to be a package-level failure, even if no
tests inside the package are considered failures.

This led to cases where the user would see an "All tests passed"
message from the go test filter, but still mysteriously fail the make
step.

To address this, the test renderer now:

  * treats package-level pass/fail/skip events as first-class citizens
    and includes them in its event count,
  * tracks the cached test output at both a package and individual test
    level, and
  * displays the whole package output if a package is marked as failed,
    but only if there is no obvious failed test top account for the
    package-level failure.

This patch also removes the json files created by the unit tests, as
they are not yet needed for anything.
This commit is contained in:
Trent Clarke 2021-10-27 11:14:27 +11:00 committed by GitHub
parent a731e3c9ff
commit eca9603376
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 138 additions and 47 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
Makefile
View file

@ -281,6 +281,7 @@ endif
.PHONY: clean
clean:
@echo "---> Cleaning up OSS build artifacts."
rm -rf build.assets/build
rm -rf $(BUILDDIR)
rm -rf $(ER_BPF_BUILDDIR)
rm -rf $(RS_BPF_BUILDDIR)
@ -441,10 +442,8 @@ test-go: PACKAGES := $(shell go list ./... | grep -v integration)
test-go: CHAOS_FOLDERS := $(shell find . -type f -name '*chaos*.go' -not -path '*/vendor/*' | xargs dirname | uniq)
test-go: $(VERSRC)
$(CGOFLAG) go test -p 4 -cover -json -tags "$(PAM_TAG) $(FIPS_TAG) $(BPF_TAG) $(ROLETESTER_TAG) $(DESKTOP_ACCESS_BETA_TAG)" $(PACKAGES) $(FLAGS) $(ADDFLAGS) \
| tee tests-unit.json \
| go run build.assets/render-tests/main.go
$(CGOFLAG) go test -p 4 -cover -json -tags "$(PAM_TAG) $(FIPS_TAG) $(BPF_TAG) $(ROLETESTER_TAG) $(DESKTOP_ACCESS_BETA_TAG)" -test.run=TestChaos $(CHAOS_FOLDERS) -cover \
| tee -a tests-unit.json \
$(CGOFLAG) go test -p 4 -cover -json -tags "$(PAM_TAG) $(FIPS_TAG) $(BPF_TAG) $(ROLETESTER_TAG) $(DESKTOP_ACCESS_BETA_TAG)" -test.run=TestChaos $(CHAOS_FOLDERS) \
| go run build.assets/render-tests/main.go
#

180
build.assets/render-tests/main.go
View file

@ -81,11 +81,82 @@ func readInput(input io.Reader, ch chan<- TestEvent) {
}
}
func main() {
type packageOutput struct {
output []string
subtestOutput map[string][]string
failedSubtests map[string][]string
}
testOutput := make(map[string][]string)
failedTests := make(map[string][]string)
actionCounts := make(map[string]int)
func (pkg *packageOutput) FailedTests() []string {
result := []string{}
for testName := range pkg.failedSubtests {
result = append(result, testName)
}
sort.Strings(result)
return result
}
type outputMap struct {
pkgs map[string]*packageOutput
actionCounts map[string]int
}
func newOutputMap() *outputMap {
return &outputMap{
pkgs: make(map[string]*packageOutput),
actionCounts: make(map[string]int),
}
}
func (m *outputMap) record(event TestEvent) {
m.actionCounts[event.Action]++
var pkgOutput *packageOutput
var exists bool
if pkgOutput, exists = m.pkgs[event.Package]; !exists {
pkgOutput = &packageOutput{
subtestOutput: make(map[string][]string),
failedSubtests: make(map[string][]string),
}
m.pkgs[event.Package] = pkgOutput
}
switch event.Action {
case actionOutput:
pkgOutput.output = append(pkgOutput.output, event.Output)
if event.Test != "" {
pkgOutput.subtestOutput[event.Test] = append(pkgOutput.subtestOutput[event.Test], event.Output)
}
case actionFail:
// If this is a single test result
if event.Test != "" {
pkgOutput.failedSubtests[event.Test] = pkgOutput.subtestOutput[event.Test]
} else {
// If this is a package result, we only want to preserve the package output if
// there are no failed subtests
if len(pkgOutput.failedSubtests) > 0 {
pkgOutput.output = nil
}
}
fallthrough
case actionPass, actionSkip:
delete(pkgOutput.subtestOutput, event.Test)
}
}
func (m *outputMap) getPkg(pkgName string) *packageOutput {
return m.pkgs[pkgName]
}
func (m *outputMap) deletePkg(pkgName string) {
delete(m.pkgs, pkgName)
}
func main() {
testOutput := newOutputMap()
failedPackages := make(map[string]*packageOutput)
coverage := make(map[string]float64)
events := make(chan TestEvent)
@ -109,20 +180,26 @@ func main() {
testName := event.FullName()
switch event.Action {
case actionOutput:
if matches := covPattern.FindStringSubmatch(event.Output); len(matches) != 0 {
value, err := strconv.ParseFloat(matches[1], 64)
if err != nil {
panic("Malformed coverage value: " + err.Error())
}
coverage[testName] = value
}
testOutput[testName] = append(testOutput[testName], event.Output)
testOutput.record(event)
case actionPass, actionFail, actionSkip:
// If this is a whole-package summary result
if event.Test == "" {
// if this is whole-package summary result
if event.Test == "" {
switch event.Action {
case actionOutput:
if matches := covPattern.FindStringSubmatch(event.Output); len(matches) != 0 {
value, err := strconv.ParseFloat(matches[1], 64)
if err != nil {
panic("Malformed coverage value: " + err.Error())
}
coverage[testName] = value
}
case actionFail:
// cache the failed test output
failedPackages[event.Package] = testOutput.getPkg(event.Package)
fallthrough
case actionPass, actionSkip:
// extract and format coverage value
covText := "------"
if covValue, ok := coverage[testName]; ok {
@ -131,20 +208,10 @@ func main() {
// only display package results as progress messages
fmt.Printf("%s %s: %s\n", covText, event.Action, event.Package)
} else {
// packages results don't count towards our test count.
actionCounts[event.Action]++
// we want to preserve the log of our failed tests for
// later examination
if event.Action == actionFail {
failedTests[testName] = testOutput[testName]
}
// Don't need this no more
testOutput.deletePkg(event.Package)
}
// we don't need the test output any more, it's either in the
// errors collection, or the test passed and we don't need to
// display it.
delete(testOutput, testName)
}
}
}
@ -152,33 +219,58 @@ func main() {
fmt.Println(separator)
fmt.Printf("%d tests passed. %d failed, %d skipped\n",
actionCounts[actionPass], actionCounts[actionFail], actionCounts[actionSkip])
testOutput.actionCounts[actionPass], testOutput.actionCounts[actionFail], testOutput.actionCounts[actionSkip])
fmt.Println(separator)
if len(failedTests) == 0 {
if len(failedPackages) == 0 {
fmt.Println("All tests pass. Yay!")
return
os.Exit(0)
}
names := make([]string, 0, len(failedTests))
for k := range failedTests {
// Generate a sorted list of package names, so that we present the
// packages that fail in a repeatable order.
names := make([]string, 0, len(failedPackages))
for k := range failedPackages {
names = append(names, k)
}
sort.Strings(names)
for _, testName := range names {
fmt.Printf("FAIL: %s\n", testName)
}
// Print a summary list of the failed tests and packages.
for _, pkgName := range names {
pkg := failedPackages[pkgName]
for _, testName := range names {
fmt.Println(separator)
fmt.Printf("TEST %s OUTPUT\n", testName)
fmt.Println(separator)
for _, l := range failedTests[testName] {
fmt.Print(l)
fmt.Printf("FAIL: %s\n", pkgName)
for testName := range pkg.failedSubtests {
fmt.Printf("FAIL: %s.%s\n", pkgName, testName)
}
}
os.Exit(1)
fmt.Println(separator)
// Print the output of each failed package or test. Note that we only print
// the package output if there is no identifiable test that caused the
// failure, as it will probably swamp the individual test output.
for _, pkgName := range names {
pkg := failedPackages[pkgName]
if len(pkg.failedSubtests) == 0 {
fmt.Printf("OUTPUT %s\n", pkgName)
fmt.Println(separator)
for _, l := range pkg.output {
fmt.Print(l)
}
fmt.Println(separator)
} else {
for _, testName := range pkg.FailedTests() {
fmt.Printf("OUTPUT %s.%s\n", pkgName, testName)
fmt.Println(separator)
for _, l := range pkg.failedSubtests[testName] {
fmt.Print(l)
}
fmt.Println(separator)
}
}
}
}