Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions acceptance/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,26 @@ The cloud run is keyed off `CLOUD_ENV`; see `getSkipReason` in `acceptance/accep

To run tests against a real workspace: `deco env run -i -n aws-prod-ucws -- <go test command>` (requires the `deco` tool and access to a test env).

### Running named tests on a PR

PR runs select only the tests the branch changed. To run more, name them in the **last** commit message of the PR, each line starting with a cloud prefix and listing one or more whitespace-separated test names:

```
empty commit to run tests

aws: TestAccept/bundle/resources/jobs
cloud: TestAccept/bundle/resources/pipelines TestAccept/cmd/fs/cp/file-to-file
gcp: TestAccept/bundle/validate/enum/DMS=true
```

- `aws:`, `azure:`, `gcp:` run the test on that cloud only; `cloud:` runs it on every cloud. Prefixes are lowercase and must start the line (surrounding whitespace is ignored). Lines with any other prefix are ignored, and so is a line with a word that does not start with `Test` (so `aws: fix flaky TestAccept retries` is prose, not a request to run the suite).
- Every named test is also selected for the local (testserver) runs, whatever its prefix.
- `TestAccept/<path>` selects every test dir that is `<path>` or under `<path>/`, with all its variants. `TestAccept/<dir>/KEY=value[/KEY=value...]` selects only the matching variants of test dir `<dir>`, as `go test` names them; each `KEY=value` must be in one of the dir's variants (per its `out.test.toml`). `TestAccept` alone selects every test. A name that matches no acceptance test (including non-acceptance test names) is logged as a warning and otherwise ignored.
- The named tests are added to the changed-test selection (`DATABRICKS_TEST_SELECT_CHANGED`, used by the PR cloud run and `./task local-select`) and to the windows/macOS PR subset (`TESTS_SELECT_SUBSET_PCT`), without counting toward their limits. Runs with neither selection (Linux PR cell, nightly) do not read the commit message: they run every test.
- On cloud, a named test still needs `Cloud = true` (and a matching `CloudEnvs`), but it runs even if it is `CloudSlow` (naming any variant of a dir lifts this for the dir).
- On pull requests, such a line makes `tools/testmask` select the tests and the cloud run even if the changed files alone would not. The merge queue ignores it.
- On the merge commit GitHub creates for a PR (`refs/pull/N/merge`), the message of the PR's head commit is used. The parser is in `acceptance/internal/selection/commit.go`.

## Authoring

To author a test,
Expand Down
40 changes: 34 additions & 6 deletions acceptance/acceptance_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -524,23 +524,41 @@ func testAccept(t *testing.T, inprocessMode bool, selectedTests []string, skipTo
// means all variants of that dir changed. Both selection.EnvVar and the subset
// selector keep these tests, so detect them at most once here.
var changedTests map[string][]string
var namedTests selection.Named
if changedLimit > 0 {
// A failed selection (e.g. no origin/main in a shallow checkout) must fail the
// run: treating it as "nothing changed" would silently skip new tests.
result, err := selection.FromGit(".", testDirsSet, changedLimit)
require.NoError(t, err)
t.Log(result.Summary())
changedTests = result.Tests()

// The tests the last commit message names run on top of the changed ones, without
// counting toward the limit.
namedCloud := ""
if cloudEnv != "" {
namedCloud = getCloudEnvBase(cloudEnv)
}
named, summary, unmatched, err := selection.FromCommitMessage(".", testDirsSet, namedCloud)
require.NoError(t, err)
namedTests = named
if len(namedTests) > 0 {
t.Log(summary)
}
for _, name := range unmatched {
t.Logf("WARNING: %q in the last commit message matches no acceptance test", name)
}
}
subset.changed = changedTests
subset.named = namedTests

// Drop the tests that were not selected instead of skipping them per dir: a skip
// per dir buries the run in a thousand SKIP lines and hides the selection summary.
// Their out.test.toml is left alone, which is what a partial run should do.
if selectChanged {
testDirs = slices.DeleteFunc(testDirs, func(dir string) bool {
_, ok := changedTests[dir]
return !ok
return !ok && !namedTests.Has(dir)
})
}

Expand Down Expand Up @@ -592,7 +610,7 @@ func testAccept(t *testing.T, inprocessMode bool, selectedTests []string, skipTo
t.Skip("Skipping test execution (only regenerating out.test.toml)")
}

skipReason := getSkipReason(&config, configPath)
skipReason := getSkipReason(&config, configPath, namedTests.Has(dir))
if skipReason != "" {
skippedDirs += 1
t.Skip(skipReason)
Expand Down Expand Up @@ -645,7 +663,7 @@ func testAccept(t *testing.T, inprocessMode bool, selectedTests []string, skipTo
for ind, envset := range expanded {
// Under selection.EnvVar, an invariant dir re-enabled by a
// specific config change runs only its matching variants.
if selectChanged && !selection.MatchesFilters(envset, changedTests[dir]) {
if selectChanged && !isSelectedVariant(changedTests, namedTests, dir, envset) {
continue
}
matched = true
Expand Down Expand Up @@ -717,8 +735,18 @@ func validateTestPhase(phase int) error {
return fmt.Errorf("Phase must be 0 or 1, got %d", phase)
}

// Return a reason to skip the test. Empty string means "don't skip".
func getSkipReason(config *internal.TestConfig, configPath string) string {
// isSelectedVariant reports whether the variant envset of dir is selected, either as a changed
// test or as one named in the last commit message.
func isSelectedVariant(changedTests map[string][]string, namedTests selection.Named, dir string, envset []string) bool {
if filters, ok := changedTests[dir]; ok && selection.MatchesFilters(envset, filters) {
return true
}
return namedTests.Matches(dir, envset)
}

// Return a reason to skip the test. Empty string means "don't skip". named is set for a test
// the last commit message names, which runs on cloud even if it is CloudSlow.
func getSkipReason(config *internal.TestConfig, configPath string, named bool) string {
if Forcerun {
return ""
}
Expand Down Expand Up @@ -747,7 +775,7 @@ func getSkipReason(config *internal.TestConfig, configPath string) string {
}

// CloudSlow only narrows an already-enabled cloud run: skip it under -short.
if isTruePtr(config.CloudSlow) && testing.Short() {
if isTruePtr(config.CloudSlow) && testing.Short() && !named {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A prefix name like cloud: TestAccept lifts CloudSlow for every dir under it, more than the -short nightly runs and inside cloud-select's 2h timeout. Should the bypass need an exact dir or variant name?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agreed. Fixed in #7548: only a name that is the dir itself or one of its variants lifts CloudSlow (new Named.Exact). A prefix such as cloud: TestAccept or TestAccept/bundle still selects the subtree, but its CloudSlow tests stay skipped under -short.

— AI agent (harness_auto, opus5.5, isaac_claude)

return fmt.Sprintf("Disabled via CloudSlow setting in %s (CLOUD_ENV=%s, Short=%v)", configPath, cloudEnv, testing.Short())
}
}
Expand Down
207 changes: 207 additions & 0 deletions acceptance/internal/selection/commit.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,207 @@
package selection

import (
"errors"
"fmt"
"os/exec"
"regexp"
"slices"
"strings"
)

// commitClouds are the prefixes a commit message line can name tests with. "cloud" names a
// test for every cloud; the others name it for that cloud only. tools/testmask/commit.go
// repeats this list (the tools module cannot import this package); keep the two in sync.
var commitClouds = []string{"aws", "azure", "gcp", "cloud"}

// commitLineRe matches a "<cloud>: <test names>" line once surrounding whitespace is trimmed.
var commitLineRe = regexp.MustCompile(`^(` + strings.Join(commitClouds, "|") + `):(.*)$`)

// githubMergeRe matches the message of the merge commit GitHub creates for refs/pull/N/merge,
// which is what pull_request CI checks out: "Merge <head sha> into <base sha>".
var githubMergeRe = regexp.MustCompile(`^Merge ([0-9a-f]{40}) into [0-9a-f]{40}$`)

// testAcceptName is the go test name of the acceptance suite. Commit message names are full go
// test names, so they start with it.
const testAcceptName = "TestAccept"

// CommitName is one test name from a commit message line, with the cloud it was named for.
type CommitName struct {
// Cloud is one of commitClouds.
Cloud string

// Name is the test name as written, e.g. TestAccept/bundle/resources/jobs.
Name string
}

// ParseCommitMessage returns the test names on the "<cloud>: <names>" lines of msg. A line may
// list several whitespace-separated names. Lines with any other prefix are ignored, and so are
// lines with a word that is not a go test name: "aws: fix flaky TestAccept retries" is prose,
// and taking its bare TestAccept as a name would run the whole suite.
func ParseCommitMessage(msg string) []CommitName {
var names []CommitName
for line := range strings.SplitSeq(msg, "\n") {
m := commitLineRe.FindStringSubmatch(strings.TrimSpace(line))
if m == nil {
continue
}
words := strings.Fields(m[2])
if len(words) == 0 || slices.ContainsFunc(words, func(word string) bool { return !strings.HasPrefix(word, "Test") }) {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A line like aws: `TestAccept/x` (backticks, or a trailing word) is dropped as prose with no warning, unlike an unmatched name. Can those lines be logged too, so a silent no-op is visible?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agreed. Fixed in #7548: a cloud-prefixed line that is not read as names (backticks, a trailing word, prose) now logs WARNING: ignoring "<line>" ... next to the unmatched-name warning, and the selection cmd prints it too.

— AI agent (harness_auto, opus5.5, isaac_claude)

continue
}
for _, name := range words {
names = append(names, CommitName{Cloud: m[1], Name: name})
}
}
return names
}

// NamesForCloud returns the names that apply to cloud (an aws/azure/gcp base cloud name), or
// every name when cloud is empty, which is a local run: there a test named for any cloud is
// also selected locally.
func NamesForCloud(names []CommitName, cloud string) []string {
var out []string
for _, n := range names {
if cloud == "" || n.Cloud == "cloud" || n.Cloud == cloud {
if !slices.Contains(out, n.Name) {
out = append(out, n.Name)
}
}
}
return out
}

// Named maps a test dir to the variant filters of each name that selected it. A nil entry
// means a name selected every variant of the dir. Names are kept apart because each one's
// filters must all hold together, while the names themselves are alternatives.
type Named map[string][][]string

// Has reports whether any name selected dir.
func (n Named) Has(dir string) bool {
_, ok := n[dir]
return ok
}

// Matches reports whether a name selected the variant envset of dir.
func (n Named) Matches(dir string, envset []string) bool {
for _, filters := range n[dir] {
if filters == nil || MatchesFilters(envset, filters) {
return true
}
}
return false
}

// add records that a name selected dir with filters, nil for every variant.
func (n Named) add(dir string, filters []string) {
n[dir] = append(n[dir], filters)
}

// ResolveNames maps go test names to the test dirs they select, and returns the names that
// selected nothing. A name selects:
// - "TestAccept": every test;
// - "TestAccept/<path>": every test dir that is <path> or under <path>/, with all variants;
// - "TestAccept/<dir>/<KEY=value>[/<KEY=value>...]": the variants of test dir <dir> that
// have those values, i.e. a variant subtest name as go test prints it. root is the
// acceptance directory, read for the variants each dir runs, so a misspelled key or value
// is reported rather than selecting every variant or none.
func ResolveNames(root string, names []string, testDirs map[string]bool) (Named, []string) {
named := Named{}
var unmatched []string
for _, name := range names {
if !resolveName(root, name, testDirs, named) {
unmatched = append(unmatched, name)
}
}
return named, unmatched
}

func resolveName(root, name string, testDirs map[string]bool, named Named) bool {
rest, ok := strings.CutPrefix(name, testAcceptName)
if !ok || (rest != "" && !strings.HasPrefix(rest, "/")) {
return false
}
rest = strings.Trim(rest, "/")

found := false
for dir := range testDirs {
if rest == "" || dir == rest || strings.HasPrefix(dir, rest+"/") {
named.add(dir, nil)
found = true
}
}
if found {
return true
}

// The longest test dir the name starts with, followed by the variant.
for i := strings.LastIndex(rest, "/"); i > 0; i = strings.LastIndex(rest[:i], "/") {
dir := rest[:i]
if !testDirs[dir] {
continue
}
filters := strings.Split(rest[i+1:], "/")
if !slices.ContainsFunc(Variants(root, dir), func(envset []string) bool { return hasAll(envset, filters) }) {
return false
}
named.add(dir, filters)
return true
}
return false
}

// hasAll reports whether every KEY=value filter is literally in envset. Unlike MatchesFilters,
// a key the envset lacks does not match, so a misspelled key selects nothing.
func hasAll(envset, filters []string) bool {
for _, filter := range filters {
if !slices.Contains(envset, filter) {
return false
}
}
return true
}

// LastCommitMessage returns the message of the commit at HEAD. On the merge commit GitHub
// creates for a pull request, it returns the message of the pull request's head commit
// instead, which is the last commit its author pushed.
func LastCommitMessage() (string, error) {
msg, err := commitMessage("HEAD")
if err != nil {
return "", err
}
if m := githubMergeRe.FindStringSubmatch(strings.TrimSpace(msg)); m != nil {
return commitMessage(m[1])
}
return msg, nil
}

func commitMessage(rev string) (string, error) {
out, err := exec.Command("git", "log", "-1", "--format=%B", rev).Output()
if err != nil {
if exitErr, ok := errors.AsType[*exec.ExitError](err); ok {
return "", fmt.Errorf("git log -1 %s failed: %w: %s", rev, err, strings.TrimSpace(string(exitErr.Stderr)))
}
return "", fmt.Errorf("git log -1 %s failed: %w", rev, err)
}
return string(out), nil
}

// FromCommitMessage selects among testDirs the tests the last commit message names for cloud
// (empty for a local run, see NamesForCloud). root is the acceptance directory. It returns the
// selection, a one-line summary of the names that matched for the test log, and the names that
// matched no test.
func FromCommitMessage(root string, testDirs map[string]bool, cloud string) (Named, string, []string, error) {
msg, err := LastCommitMessage()
if err != nil {
return nil, "", nil, err
}
names := NamesForCloud(ParseCommitMessage(msg), cloud)
named, unmatched := ResolveNames(root, names, testDirs)
matched := slices.DeleteFunc(names, func(name string) bool { return slices.Contains(unmatched, name) })
where := "local"
if cloud != "" {
where = cloud
}
summary := fmt.Sprintf("Selected %d test dirs named in the last commit message (%s): %s", len(named), where, strings.Join(matched, " "))
return named, summary, unmatched, nil
}
Loading
Loading