diff --git a/.github/workflows/regression-report.yml b/.github/workflows/regression-report.yml index 83793007d98..6c4052a1136 100644 --- a/.github/workflows/regression-report.yml +++ b/.github/workflows/regression-report.yml @@ -60,7 +60,7 @@ jobs: owner, repo, issue_number: context.issue.number, }); const existing = comments.find(c => - c.user.login === 'github-actions[bot]' && c.body && c.body.startsWith('')); + c.user.login === 'github-actions[bot]' && c.body && c.body.includes('## Regression test report')); if (existing) { await github.rest.issues.deleteComment({ owner, repo, comment_id: existing.id }); } @@ -122,14 +122,14 @@ jobs: const path = process.env.GITHUB_WORKSPACE + '/regression_report.md'; // Other comments by the same bot are left alone: ours is found by this marker. - const MARKER = ''; + const HEADING = '## Regression test report'; const { owner, repo } = context.repo; const issue_number = context.issue.number; const comments = await github.paginate(github.rest.issues.listComments, { owner, repo, issue_number, }); const existing = comments.find(c => - c.user.login === 'github-actions[bot]' && c.body && c.body.startsWith(MARKER)); + c.user.login === 'github-actions[bot]' && c.body && c.body.includes(HEADING)); if (!fs.existsSync(path)) { if (existing) { @@ -160,7 +160,7 @@ jobs: report = kept + `_${blocks.length - shown} more failure details omitted, see the full report in the [run](${runUrl})._\n`; } - const body = `${MARKER}\nCommit: [\`${sha.slice(0, 9)}\`](${commitUrl}) | [Workflow run](${runUrl})\n\n${report}`; + const body = report.replace('\n\n', `\n\nCommit: [\`${sha.slice(0, 9)}\`](${commitUrl}) | [Workflow run](${runUrl})\n\n`); if (existing) { await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body }); diff --git a/acceptance/regression_test.go b/acceptance/regression_test.go index bcae5a96a84..925e3649a14 100644 --- a/acceptance/regression_test.go +++ b/acceptance/regression_test.go @@ -222,14 +222,19 @@ func formatCounts(counts map[string]int) string { func renderRegressionReport(selected string, columns []regrColumn) string { branch, refs := columns[0], columns[1:] tests := leafTests(branch.Results) + counts := countByCategory(tests, branch, refs) + status := statusPass + if counts[catFailing] > 0 { + status = statusFail + } var b strings.Builder - b.WriteString("## Regression test report\n\n") + fmt.Fprintf(&b, "## Regression test report %s\n\n", statusMark(status)) fmt.Fprintf(&b, "%s\n\n", selected) // The table has a row per test variant, often hundreds, so it is collapsed. The workflow // that posts the report always keeps this first
block when it trims the comment. - fmt.Fprintf(&b, "
\n%d tests: %s\n\n", len(tests), formatCounts(countByCategory(tests, branch, refs))) + fmt.Fprintf(&b, "
\n%d tests: %s\n\n", len(tests), formatCounts(counts)) b.WriteString("| test | " + strings.Join(columnTitles(columns), " | ") + " |\n") b.WriteString("|---|" + strings.Repeat("---|", len(columns)) + "\n") for _, test := range tests { diff --git a/acceptance/regression_unit_test.go b/acceptance/regression_unit_test.go index 4583e04d2b2..54fd030de9c 100644 --- a/acceptance/regression_unit_test.go +++ b/acceptance/regression_unit_test.go @@ -69,6 +69,39 @@ func TestRenderRegressionReport(t *testing.T) { assert.Contains(t, report, "failing=1") } +func TestRenderRegressionReportHeading(t *testing.T) { + testCases := []struct { + name string + branchStatus string + mainStatus string + want string + }{ + {"passed", statusPass, statusPass, "✅"}, + {"main failed", statusPass, statusFail, "✅"}, + {"skipped", statusSkip, statusPass, "✅"}, + {"branch failed", statusFail, statusPass, "❌"}, + {"both failed", statusFail, statusFail, "❌"}, + } + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + columns := []regrColumn{ + {Title: "branch", Results: map[string]*testResult{ + "TestAccept/bundle/a": {Status: statusPass}, + "TestAccept/bundle/b": {Status: testCase.branchStatus}, + }}, + {Title: "main", Category: catRegression, Results: map[string]*testResult{ + "TestAccept/bundle/a": {Status: statusPass}, + "TestAccept/bundle/b": {Status: testCase.mainStatus}, + }}, + } + + report := renderRegressionReport("Selected 2", columns) + heading, _, _ := strings.Cut(report, "\n") + assert.Equal(t, "## Regression test report "+testCase.want, heading) + }) + } +} + func TestRenderRegressionReportDedupesIdenticalFailures(t *testing.T) { failure := func(tmp string) *testResult { return &testResult{Status: statusFail, Output: []string{"Messages: output.txt vs /tmp/TestAcceptfoo" + tmp + "/002/output.txt", "diff"}}