diff --git a/acceptance/README.md b/acceptance/README.md index 9cd4d2ed0a0..fe261072b25 100644 --- a/acceptance/README.md +++ b/acceptance/README.md @@ -15,6 +15,26 @@ The cloud run is keyed off `CLOUD_ENV`; see `getSkipReason` in `acceptance/accep To run tests against a real workspace: `deco env run -i -n aws-prod-ucws -- ` (requires the `deco` tool and access to a test env). +### Running named tests on a PR + +PR runs select only the tests the branch changed. To run more, name them in the **last** commit message of the PR, each line starting with a cloud prefix and listing one or more whitespace-separated test names: + +``` +empty commit to run tests + +aws: TestAccept/bundle/resources/jobs +cloud: TestAccept/bundle/resources/pipelines TestAccept/cmd/fs/cp/file-to-file +gcp: TestAccept/bundle/validate/enum/DMS=true +``` + +- `aws:`, `azure:`, `gcp:` run the test on that cloud only; `cloud:` runs it on every cloud. Prefixes are lowercase and must start the line (surrounding whitespace is ignored). Lines with any other prefix are ignored, and so is a line with a word that does not start with `Test` (so `aws: fix flaky TestAccept retries` is prose, not a request to run the suite). +- Every named test is also selected for the local (testserver) runs, whatever its prefix. +- `TestAccept/` selects every test dir that is `` or under `/`, with all its variants. `TestAccept//KEY=value[/KEY=value...]` selects only the matching variants of test dir ``, as `go test` names them; each `KEY=value` must be in one of the dir's variants (per its `out.test.toml`). `TestAccept` alone selects every test. A name that matches no acceptance test (including non-acceptance test names) is logged as a warning and otherwise ignored. +- The named tests are added to the changed-test selection (`DATABRICKS_TEST_SELECT_CHANGED`, used by the PR cloud run and `./task local-select`) and to the windows/macOS PR subset (`TESTS_SELECT_SUBSET_PCT`), without counting toward their limits. Runs with neither selection (Linux PR cell, nightly) do not read the commit message: they run every test. +- On cloud, a named test still needs `Cloud = true` (and a matching `CloudEnvs`), but it runs even if it is `CloudSlow` (naming any variant of a dir lifts this for the dir). +- On pull requests, such a line makes `tools/testmask` select the tests and the cloud run even if the changed files alone would not. The merge queue ignores it. +- On the merge commit GitHub creates for a PR (`refs/pull/N/merge`), the message of the PR's head commit is used. The parser is in `acceptance/internal/selection/commit.go`. + ## Authoring To author a test, diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 3805300a8e7..f5ac54874a0 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -524,6 +524,7 @@ func testAccept(t *testing.T, inprocessMode bool, selectedTests []string, skipTo // means all variants of that dir changed. Both selection.EnvVar and the subset // selector keep these tests, so detect them at most once here. var changedTests map[string][]string + var namedTests selection.Named if changedLimit > 0 { // A failed selection (e.g. no origin/main in a shallow checkout) must fail the // run: treating it as "nothing changed" would silently skip new tests. @@ -531,8 +532,25 @@ func testAccept(t *testing.T, inprocessMode bool, selectedTests []string, skipTo require.NoError(t, err) t.Log(result.Summary()) changedTests = result.Tests() + + // The tests the last commit message names run on top of the changed ones, without + // counting toward the limit. + namedCloud := "" + if cloudEnv != "" { + namedCloud = getCloudEnvBase(cloudEnv) + } + named, summary, unmatched, err := selection.FromCommitMessage(".", testDirsSet, namedCloud) + require.NoError(t, err) + namedTests = named + if len(namedTests) > 0 { + t.Log(summary) + } + for _, name := range unmatched { + t.Logf("WARNING: %q in the last commit message matches no acceptance test", name) + } } subset.changed = changedTests + subset.named = namedTests // Drop the tests that were not selected instead of skipping them per dir: a skip // per dir buries the run in a thousand SKIP lines and hides the selection summary. @@ -540,7 +558,7 @@ func testAccept(t *testing.T, inprocessMode bool, selectedTests []string, skipTo if selectChanged { testDirs = slices.DeleteFunc(testDirs, func(dir string) bool { _, ok := changedTests[dir] - return !ok + return !ok && !namedTests.Has(dir) }) } @@ -592,7 +610,7 @@ func testAccept(t *testing.T, inprocessMode bool, selectedTests []string, skipTo t.Skip("Skipping test execution (only regenerating out.test.toml)") } - skipReason := getSkipReason(&config, configPath) + skipReason := getSkipReason(&config, configPath, namedTests.Has(dir)) if skipReason != "" { skippedDirs += 1 t.Skip(skipReason) @@ -645,7 +663,7 @@ func testAccept(t *testing.T, inprocessMode bool, selectedTests []string, skipTo for ind, envset := range expanded { // Under selection.EnvVar, an invariant dir re-enabled by a // specific config change runs only its matching variants. - if selectChanged && !selection.MatchesFilters(envset, changedTests[dir]) { + if selectChanged && !isSelectedVariant(changedTests, namedTests, dir, envset) { continue } matched = true @@ -717,8 +735,18 @@ func validateTestPhase(phase int) error { return fmt.Errorf("Phase must be 0 or 1, got %d", phase) } -// Return a reason to skip the test. Empty string means "don't skip". -func getSkipReason(config *internal.TestConfig, configPath string) string { +// isSelectedVariant reports whether the variant envset of dir is selected, either as a changed +// test or as one named in the last commit message. +func isSelectedVariant(changedTests map[string][]string, namedTests selection.Named, dir string, envset []string) bool { + if filters, ok := changedTests[dir]; ok && selection.MatchesFilters(envset, filters) { + return true + } + return namedTests.Matches(dir, envset) +} + +// Return a reason to skip the test. Empty string means "don't skip". named is set for a test +// the last commit message names, which runs on cloud even if it is CloudSlow. +func getSkipReason(config *internal.TestConfig, configPath string, named bool) string { if Forcerun { return "" } @@ -747,7 +775,7 @@ func getSkipReason(config *internal.TestConfig, configPath string) string { } // CloudSlow only narrows an already-enabled cloud run: skip it under -short. - if isTruePtr(config.CloudSlow) && testing.Short() { + if isTruePtr(config.CloudSlow) && testing.Short() && !named { return fmt.Sprintf("Disabled via CloudSlow setting in %s (CLOUD_ENV=%s, Short=%v)", configPath, cloudEnv, testing.Short()) } } diff --git a/acceptance/internal/selection/commit.go b/acceptance/internal/selection/commit.go new file mode 100644 index 00000000000..353c5c71950 --- /dev/null +++ b/acceptance/internal/selection/commit.go @@ -0,0 +1,207 @@ +package selection + +import ( + "errors" + "fmt" + "os/exec" + "regexp" + "slices" + "strings" +) + +// commitClouds are the prefixes a commit message line can name tests with. "cloud" names a +// test for every cloud; the others name it for that cloud only. tools/testmask/commit.go +// repeats this list (the tools module cannot import this package); keep the two in sync. +var commitClouds = []string{"aws", "azure", "gcp", "cloud"} + +// commitLineRe matches a ": " line once surrounding whitespace is trimmed. +var commitLineRe = regexp.MustCompile(`^(` + strings.Join(commitClouds, "|") + `):(.*)$`) + +// githubMergeRe matches the message of the merge commit GitHub creates for refs/pull/N/merge, +// which is what pull_request CI checks out: "Merge into ". +var githubMergeRe = regexp.MustCompile(`^Merge ([0-9a-f]{40}) into [0-9a-f]{40}$`) + +// testAcceptName is the go test name of the acceptance suite. Commit message names are full go +// test names, so they start with it. +const testAcceptName = "TestAccept" + +// CommitName is one test name from a commit message line, with the cloud it was named for. +type CommitName struct { + // Cloud is one of commitClouds. + Cloud string + + // Name is the test name as written, e.g. TestAccept/bundle/resources/jobs. + Name string +} + +// ParseCommitMessage returns the test names on the ": " lines of msg. A line may +// list several whitespace-separated names. Lines with any other prefix are ignored, and so are +// lines with a word that is not a go test name: "aws: fix flaky TestAccept retries" is prose, +// and taking its bare TestAccept as a name would run the whole suite. +func ParseCommitMessage(msg string) []CommitName { + var names []CommitName + for line := range strings.SplitSeq(msg, "\n") { + m := commitLineRe.FindStringSubmatch(strings.TrimSpace(line)) + if m == nil { + continue + } + words := strings.Fields(m[2]) + if len(words) == 0 || slices.ContainsFunc(words, func(word string) bool { return !strings.HasPrefix(word, "Test") }) { + continue + } + for _, name := range words { + names = append(names, CommitName{Cloud: m[1], Name: name}) + } + } + return names +} + +// NamesForCloud returns the names that apply to cloud (an aws/azure/gcp base cloud name), or +// every name when cloud is empty, which is a local run: there a test named for any cloud is +// also selected locally. +func NamesForCloud(names []CommitName, cloud string) []string { + var out []string + for _, n := range names { + if cloud == "" || n.Cloud == "cloud" || n.Cloud == cloud { + if !slices.Contains(out, n.Name) { + out = append(out, n.Name) + } + } + } + return out +} + +// Named maps a test dir to the variant filters of each name that selected it. A nil entry +// means a name selected every variant of the dir. Names are kept apart because each one's +// filters must all hold together, while the names themselves are alternatives. +type Named map[string][][]string + +// Has reports whether any name selected dir. +func (n Named) Has(dir string) bool { + _, ok := n[dir] + return ok +} + +// Matches reports whether a name selected the variant envset of dir. +func (n Named) Matches(dir string, envset []string) bool { + for _, filters := range n[dir] { + if filters == nil || MatchesFilters(envset, filters) { + return true + } + } + return false +} + +// add records that a name selected dir with filters, nil for every variant. +func (n Named) add(dir string, filters []string) { + n[dir] = append(n[dir], filters) +} + +// ResolveNames maps go test names to the test dirs they select, and returns the names that +// selected nothing. A name selects: +// - "TestAccept": every test; +// - "TestAccept/": every test dir that is or under /, with all variants; +// - "TestAccept//[/...]": the variants of test dir that +// have those values, i.e. a variant subtest name as go test prints it. root is the +// acceptance directory, read for the variants each dir runs, so a misspelled key or value +// is reported rather than selecting every variant or none. +func ResolveNames(root string, names []string, testDirs map[string]bool) (Named, []string) { + named := Named{} + var unmatched []string + for _, name := range names { + if !resolveName(root, name, testDirs, named) { + unmatched = append(unmatched, name) + } + } + return named, unmatched +} + +func resolveName(root, name string, testDirs map[string]bool, named Named) bool { + rest, ok := strings.CutPrefix(name, testAcceptName) + if !ok || (rest != "" && !strings.HasPrefix(rest, "/")) { + return false + } + rest = strings.Trim(rest, "/") + + found := false + for dir := range testDirs { + if rest == "" || dir == rest || strings.HasPrefix(dir, rest+"/") { + named.add(dir, nil) + found = true + } + } + if found { + return true + } + + // The longest test dir the name starts with, followed by the variant. + for i := strings.LastIndex(rest, "/"); i > 0; i = strings.LastIndex(rest[:i], "/") { + dir := rest[:i] + if !testDirs[dir] { + continue + } + filters := strings.Split(rest[i+1:], "/") + if !slices.ContainsFunc(Variants(root, dir), func(envset []string) bool { return hasAll(envset, filters) }) { + return false + } + named.add(dir, filters) + return true + } + return false +} + +// hasAll reports whether every KEY=value filter is literally in envset. Unlike MatchesFilters, +// a key the envset lacks does not match, so a misspelled key selects nothing. +func hasAll(envset, filters []string) bool { + for _, filter := range filters { + if !slices.Contains(envset, filter) { + return false + } + } + return true +} + +// LastCommitMessage returns the message of the commit at HEAD. On the merge commit GitHub +// creates for a pull request, it returns the message of the pull request's head commit +// instead, which is the last commit its author pushed. +func LastCommitMessage() (string, error) { + msg, err := commitMessage("HEAD") + if err != nil { + return "", err + } + if m := githubMergeRe.FindStringSubmatch(strings.TrimSpace(msg)); m != nil { + return commitMessage(m[1]) + } + return msg, nil +} + +func commitMessage(rev string) (string, error) { + out, err := exec.Command("git", "log", "-1", "--format=%B", rev).Output() + if err != nil { + if exitErr, ok := errors.AsType[*exec.ExitError](err); ok { + return "", fmt.Errorf("git log -1 %s failed: %w: %s", rev, err, strings.TrimSpace(string(exitErr.Stderr))) + } + return "", fmt.Errorf("git log -1 %s failed: %w", rev, err) + } + return string(out), nil +} + +// FromCommitMessage selects among testDirs the tests the last commit message names for cloud +// (empty for a local run, see NamesForCloud). root is the acceptance directory. It returns the +// selection, a one-line summary of the names that matched for the test log, and the names that +// matched no test. +func FromCommitMessage(root string, testDirs map[string]bool, cloud string) (Named, string, []string, error) { + msg, err := LastCommitMessage() + if err != nil { + return nil, "", nil, err + } + names := NamesForCloud(ParseCommitMessage(msg), cloud) + named, unmatched := ResolveNames(root, names, testDirs) + matched := slices.DeleteFunc(names, func(name string) bool { return slices.Contains(unmatched, name) }) + where := "local" + if cloud != "" { + where = cloud + } + summary := fmt.Sprintf("Selected %d test dirs named in the last commit message (%s): %s", len(named), where, strings.Join(matched, " ")) + return named, summary, unmatched, nil +} diff --git a/acceptance/internal/selection/commit_test.go b/acceptance/internal/selection/commit_test.go new file mode 100644 index 00000000000..66e962769d3 --- /dev/null +++ b/acceptance/internal/selection/commit_test.go @@ -0,0 +1,175 @@ +package selection + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseCommitMessage(t *testing.T) { + msg := "empty commit to run tests\n" + + "\n" + + "aws: TestAccept/bundle/resources/jobs\n" + + "cloud: TestAccept/bundle/resources/pipelines \n" + + " gcp:TestAccept/a TestAccept/b\t\n" + + "azure: TestAccept/c/DMS=true\r\n" + + "dbr: TestAccept/unknown/prefix\n" + + "AWS: TestAccept/uppercase/prefix\n" + + "see aws: TestAccept/not/at/line/start\n" + + "aws: fix flaky TestAccept retries\n" + + "Co-authored-by: Someone \n" + + "aws:\n" + + assert.Equal(t, []CommitName{ + {Cloud: "aws", Name: "TestAccept/bundle/resources/jobs"}, + {Cloud: "cloud", Name: "TestAccept/bundle/resources/pipelines"}, + {Cloud: "gcp", Name: "TestAccept/a"}, + {Cloud: "gcp", Name: "TestAccept/b"}, + {Cloud: "azure", Name: "TestAccept/c/DMS=true"}, + }, ParseCommitMessage(msg)) +} + +func TestParseCommitMessageNoMatches(t *testing.T) { + assert.Empty(t, ParseCommitMessage("")) + assert.Empty(t, ParseCommitMessage("Fix a bug\n\nThe aws: prefix is mentioned mid-line only.\n")) + assert.Empty(t, ParseCommitMessage("cloud: make TestAccept faster\n")) +} + +func TestNamesForCloud(t *testing.T) { + names := []CommitName{ + {Cloud: "aws", Name: "TestAccept/a"}, + {Cloud: "cloud", Name: "TestAccept/b"}, + {Cloud: "gcp", Name: "TestAccept/c"}, + {Cloud: "azure", Name: "TestAccept/d"}, + {Cloud: "gcp", Name: "TestAccept/b"}, + } + assert.Equal(t, []string{"TestAccept/a", "TestAccept/b"}, NamesForCloud(names, "aws")) + assert.Equal(t, []string{"TestAccept/b", "TestAccept/c"}, NamesForCloud(names, "gcp")) + assert.Equal(t, []string{"TestAccept/b", "TestAccept/d"}, NamesForCloud(names, "azure")) + // A local run selects the tests named for any cloud. + assert.Equal(t, []string{"TestAccept/a", "TestAccept/b", "TestAccept/c", "TestAccept/d"}, NamesForCloud(names, "")) + assert.Empty(t, NamesForCloud(nil, "aws")) +} + +func TestResolveNames(t *testing.T) { + root := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(root, "cmd", "version"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(root, "cmd", "version", "out.test.toml"), []byte("EnvMatrix.DMS = [\"\", \"true\"]\nEnvMatrix.X = [\"1\", \"2\"]\n"), 0o600)) + + testDirs := map[string]bool{ + "bundle/resources/jobs": true, + "bundle/resources/jobs/update": true, + "bundle/resources/jobsx": true, + "bundle/resources/pipelines/basic": true, + "cmd/version": true, + } + + named, unmatched := ResolveNames(root, []string{ + // A dir and everything under it, but not a sibling that shares the prefix. + "TestAccept/bundle/resources/jobs", + // A path that is not itself a test dir selects the dirs under it. + "TestAccept/bundle/resources/pipelines/", + // A variant of a dir. + "TestAccept/cmd/version/DMS=true/X=1", + "TestAccept/cmd/version/DMS=", + "TestAccept/bundle/nothing", + "TestAccept/cmd/version/notavariant", + // A misspelled key or value matches no variant. + "TestAccept/cmd/version/DSM=true", + "TestAccept/cmd/version/DMS=ture", + // A dir without a matrix has no variants to name. + "TestAccept/bundle/resources/jobsx/DMS=true", + "TestAcceptance/cmd/version", + "TestFoo", + }, testDirs) + + assert.Equal(t, Named{ + "bundle/resources/jobs": {nil}, + "bundle/resources/jobs/update": {nil}, + "bundle/resources/pipelines/basic": {nil}, + "cmd/version": {{"DMS=true", "X=1"}, {"DMS="}}, + }, named) + assert.Equal(t, []string{ + "TestAccept/bundle/nothing", + "TestAccept/cmd/version/notavariant", + "TestAccept/cmd/version/DSM=true", + "TestAccept/cmd/version/DMS=ture", + "TestAccept/bundle/resources/jobsx/DMS=true", + "TestAcceptance/cmd/version", + "TestFoo", + }, unmatched) + + all, unmatched := ResolveNames(root, []string{"TestAccept"}, testDirs) + assert.Len(t, all, len(testDirs)) + assert.Empty(t, unmatched) +} + +func TestNamedMatches(t *testing.T) { + named := Named{ + "all": {nil}, + "variant": {{"DMS=true", "X=1"}, {"DMS="}}, + } + assert.True(t, named.Has("all")) + assert.False(t, named.Has("other")) + + assert.True(t, named.Matches("all", []string{"DMS=true"})) + assert.True(t, named.Matches("variant", []string{"DMS=true", "X=1"})) + assert.True(t, named.Matches("variant", []string{"DMS=", "X=2"})) + // Each name's filters hold together: DMS=true alone was not named with X=2. + assert.False(t, named.Matches("variant", []string{"DMS=true", "X=2"})) + assert.False(t, named.Matches("other", nil)) + + var empty Named + assert.False(t, empty.Has("all")) + assert.False(t, empty.Matches("all", nil)) +} + +func TestLastCommitMessage(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skipf("git not available: %v", err) + } + t.Chdir(t.TempDir()) + + git := func(args ...string) string { + t.Helper() + out, err := exec.Command("git", args...).CombinedOutput() + require.NoError(t, err, "git %v\n%s", args, out) + return string(out) + } + + git("init", "-q", "-b", "main") + git("config", "user.email", "test@example.invalid") + git("config", "user.name", "Test") + require.NoError(t, os.WriteFile("a.txt", []byte("a"), 0o600)) + git("add", "a.txt") + git("commit", "-q", "-m", "base") + base := git("rev-parse", "HEAD") + + git("checkout", "-q", "-b", "feature") + git("commit", "-q", "--allow-empty", "-m", "run tests\n\naws: TestAccept/x") + msg, err := LastCommitMessage() + require.NoError(t, err) + assert.Equal(t, "run tests\n\naws: TestAccept/x", strings.TrimSpace(msg)) + head := git("rev-parse", "HEAD") + + // The merge commit GitHub creates for refs/pull/N/merge: the message of the PR's head + // commit is read instead of "Merge into ". + git("checkout", "-q", "main") + git("commit", "-q", "--allow-empty", "-m", "main advances") + git("merge", "-q", "--no-ff", "feature", "-m", "Merge "+head[:40]+" into "+base[:40]) + msg, err = LastCommitMessage() + require.NoError(t, err) + assert.Equal(t, "run tests\n\naws: TestAccept/x", strings.TrimSpace(msg)) + + // Any other merge commit is the last commit itself. + git("checkout", "-q", "feature") + git("merge", "-q", "--no-ff", "main", "-m", "Merge branch 'main' into feature") + msg, err = LastCommitMessage() + require.NoError(t, err) + assert.Equal(t, "Merge branch 'main' into feature", strings.TrimSpace(msg)) +} diff --git a/acceptance/subset_test.go b/acceptance/subset_test.go index b996f87cf13..f9796f43d4d 100644 --- a/acceptance/subset_test.go +++ b/acceptance/subset_test.go @@ -45,6 +45,9 @@ type subsetSelector struct { // changed maps test dir -> variant filters for added/modified tests (nil filters // means all variants of that dir are changed). Same shape as changedTests. changed map[string][]string + + // named holds the tests the last commit message names, also always kept. + named selection.Named } // newSubsetSelector reads the subset env vars. Subsetting is disabled in update mode @@ -82,7 +85,7 @@ func (s subsetSelector) skipReason(dir string, envset []string) string { if !s.enabled { return "" } - if s.isChanged(dir, envset) { + if s.isChanged(dir, envset) || s.named.Matches(dir, envset) { return "" } id := dir diff --git a/tools/testmask/commit.go b/tools/testmask/commit.go new file mode 100644 index 00000000000..bb347f27dcc --- /dev/null +++ b/tools/testmask/commit.go @@ -0,0 +1,43 @@ +package main + +import ( + "fmt" + "os/exec" + "regexp" + "slices" + "strings" +) + +// commitTestLineRe matches the prefix of a commit message line that names acceptance tests to +// run on cloud, e.g. "aws: TestAccept/bundle/resources/jobs". It mirrors the parser in +// acceptance/internal/selection/commit.go, which decides what such a line selects (the tools +// module cannot import it); testmask only needs to know that the line is there. +var commitTestLineRe = regexp.MustCompile(`^(aws|azure|gcp|cloud):(.*)$`) + +// GetCommitMessage returns the message of the commit at ref. +func GetCommitMessage(ref string) (string, error) { + out, err := exec.Command("git", "log", "-1", "--format=%B", ref).Output() + if err != nil { + return "", fmt.Errorf("failed to read commit message of %s: %w", ref, err) + } + return string(out), nil +} + +// NamesTests reports whether message has a line naming acceptance tests to run on cloud: a +// cloud prefix followed only by go test names, at least one of them an acceptance test. +func NamesTests(message string) bool { + for line := range strings.SplitSeq(message, "\n") { + m := commitTestLineRe.FindStringSubmatch(strings.TrimSpace(line)) + if m == nil { + continue + } + words := strings.Fields(m[2]) + if slices.ContainsFunc(words, func(word string) bool { return !strings.HasPrefix(word, "Test") }) { + continue + } + if slices.ContainsFunc(words, func(word string) bool { return word == "TestAccept" || strings.HasPrefix(word, "TestAccept/") }) { + return true + } + } + return false +} diff --git a/tools/testmask/commit_test.go b/tools/testmask/commit_test.go new file mode 100644 index 00000000000..e2c11c06cf3 --- /dev/null +++ b/tools/testmask/commit_test.go @@ -0,0 +1,34 @@ +package main + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNamesTests(t *testing.T) { + for _, msg := range []string{ + "run tests\n\naws: TestAccept/bundle/resources/jobs\n", + "cloud: TestAccept/a", + "gcp:TestAccept/a", + " azure: TestAccept/a \n", + "aws: TestAccept", + "aws: TestFoo TestAccept/a", + } { + assert.True(t, NamesTests(msg), msg) + } + + for _, msg := range []string{ + "", + "Fix a bug", + "dbr: TestAccept/a", + "AWS: TestAccept/a", + "see aws: TestAccept/a", + "aws: TestAcceptance/a", + "aws: TestFoo", + "aws: fix flaky TestAccept retries", + "aws:", + } { + assert.False(t, NamesTests(msg), msg) + } +} diff --git a/tools/testmask/main.go b/tools/testmask/main.go index 29737cc9e7d..7b7df71c17c 100644 --- a/tools/testmask/main.go +++ b/tools/testmask/main.go @@ -1,11 +1,13 @@ // Command testmask reads Taskfile.yml to decide which CI jobs should run -// based on the set of files changed in a PR. +// based on the set of files changed in a PR and on the tests its head commit +// message names. package main import ( "encoding/json" "fmt" "os" + "slices" ) func main() { @@ -17,6 +19,11 @@ func main() { headRef := os.Args[1] baseRef := os.Args[2] + // Only pull requests read the commit message: a merge queue commit can carry the PR's + // message, and its cloud run is skipped anyway. GITHUB_EVENT_NAME is set by GitHub Actions; + // set it to pull_request to try this locally. + readCommitMessage := os.Getenv("GITHUB_EVENT_NAME") == "pull_request" //nolint:forbidigo // the tools module cannot import libs/env + mappings, err := LoadTargetMappings("../../Taskfile.yml") if err != nil { fmt.Fprintf(os.Stderr, "Error loading target mappings: %v\n", err) @@ -30,6 +37,19 @@ func main() { } targets := GetTargets(changedFiles, mappings) + + // Tests named in the head commit message run on cloud, which only happens when the + // "test" target is selected. An unreadable message is not worth failing the run over; + // the diff still decides. + if readCommitMessage && !slices.Contains(targets, "test") { + message, err := GetCommitMessage(headRef) + if err != nil { + fmt.Fprintf(os.Stderr, "warning: ignoring commit message: %v\n", err) + } else if NamesTests(message) { + targets = append(targets, "test") + slices.Sort(targets) + } + } err = json.NewEncoder(os.Stdout).Encode(targets) if err != nil { fmt.Fprintf(os.Stderr, "Error encoding targets: %v\n", err)