Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions acceptance/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,14 @@ cloud: TestAccept/bundle/resources/pipelines TestAccept/cmd/fs/cp/file-to-file
gcp: TestAccept/bundle/validate/enum/DMS=true
```

- `aws:`, `azure:`, `gcp:` run the test on that cloud only; `cloud:` runs it on every cloud. Prefixes are lowercase and must start the line (surrounding whitespace is ignored). Lines with any other prefix are ignored, and so is a line with a word that does not start with `Test` (so `aws: fix flaky TestAccept retries` is prose, not a request to run the suite).
- `aws:`, `azure:`, `gcp:` run the test on that cloud only; `cloud:` runs it on every cloud. Prefixes are lowercase and must start the line (surrounding whitespace is ignored). Lines with any other prefix are ignored. A line with a word that does not start with `Test` is not read either (so `aws: fix flaky TestAccept retries` is prose, not a request to run the suite), and is logged as a warning, as is ``aws: `TestAccept/x` ``.
- Every named test is also selected for the local (testserver) runs, whatever its prefix.
- `TestAccept/<path>` selects every test dir that is `<path>` or under `<path>/`, with all its variants. `TestAccept/<dir>/KEY=value[/KEY=value...]` selects only the matching variants of test dir `<dir>`, as `go test` names them; each `KEY=value` must be in one of the dir's variants (per its `out.test.toml`). `TestAccept` alone selects every test. A name that matches no acceptance test (including non-acceptance test names) is logged as a warning and otherwise ignored.
- The named tests are added to the changed-test selection (`DATABRICKS_TEST_SELECT_CHANGED`, used by the PR cloud run and `./task local-select`) and to the windows/macOS PR subset (`TESTS_SELECT_SUBSET_PCT`), without counting toward their limits. Runs with neither selection (Linux PR cell, nightly) do not read the commit message: they run every test.
- On cloud, a named test still needs `Cloud = true` (and a matching `CloudEnvs`), but it runs even if it is `CloudSlow` (naming any variant of a dir lifts this for the dir).
- On cloud, a named test still needs `Cloud = true` (and a matching `CloudEnvs`), but it runs even if it is `CloudSlow` when it is named by its own dir or variant name (naming any variant of a dir lifts this for the dir). A path above the dir (or a bare `TestAccept`) does not lift `CloudSlow`, so a whole subtree stays within the PR run's `-short` budget.
- On pull requests, such a line makes `tools/testmask` select the tests and the cloud run even if the changed files alone would not. The merge queue ignores it.
- On the merge commit GitHub creates for a PR (`refs/pull/N/merge`), the message of the PR's head commit is used. The parser is in `acceptance/internal/selection/commit.go`.
- To see what a commit message selects without running the suite: `go run ./acceptance/internal/selection/cmd [-cloud aws|azure|gcp]` prints the named tests and the warnings after the changed ones.

## Authoring

Expand Down
15 changes: 8 additions & 7 deletions acceptance/acceptance_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -539,14 +539,14 @@ func testAccept(t *testing.T, inprocessMode bool, selectedTests []string, skipTo
if cloudEnv != "" {
namedCloud = getCloudEnvBase(cloudEnv)
}
named, summary, unmatched, err := selection.FromCommitMessage(".", testDirsSet, namedCloud)
commit, err := selection.FromCommitMessage(".", testDirsSet, namedCloud)
require.NoError(t, err)
namedTests = named
namedTests = commit.Named
if len(namedTests) > 0 {
t.Log(summary)
t.Log(commit.Summary())
}
for _, name := range unmatched {
t.Logf("WARNING: %q in the last commit message matches no acceptance test", name)
for _, warning := range commit.Warnings() {
t.Log("WARNING: " + warning)
}
}
subset.changed = changedTests
Expand Down Expand Up @@ -610,7 +610,7 @@ func testAccept(t *testing.T, inprocessMode bool, selectedTests []string, skipTo
t.Skip("Skipping test execution (only regenerating out.test.toml)")
}

skipReason := getSkipReason(&config, configPath, namedTests.Has(dir))
skipReason := getSkipReason(&config, configPath, namedTests.Exact(dir))
if skipReason != "" {
skippedDirs += 1
t.Skip(skipReason)
Expand Down Expand Up @@ -745,7 +745,8 @@ func isSelectedVariant(changedTests map[string][]string, namedTests selection.Na
}

// Return a reason to skip the test. Empty string means "don't skip". named is set for a test
// the last commit message names, which runs on cloud even if it is CloudSlow.
// the last commit message names by its own name (not by a path above it), which runs on cloud
// even if it is CloudSlow.
func getSkipReason(config *internal.TestConfig, configPath string, named bool) string {
if Forcerun {
return ""
Expand Down
59 changes: 54 additions & 5 deletions acceptance/internal/selection/cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,14 @@
//
// go run ./acceptance/internal/selection/cmd
// go run ./acceptance/internal/selection/cmd -limit 5
// go run ./acceptance/internal/selection/cmd -cloud aws
// go run ./acceptance/internal/selection/cmd acceptance/bundle/invariant/configs/job.yml.tmpl
//
// With no arguments the selection comes from git, exactly as the test runner gets it. Each
// argument stands for a changed file instead, and counts as modified unless it carries a
// status ("A:path" for an added file, "R100:old:new" for a rename).
// With no arguments the selection comes from git, exactly as the test runner gets it, and
// also lists the tests the last commit message names (see acceptance/README.md) for the run
// -cloud picks: aws, azure or gcp, or empty for a local run. Each argument stands for a
// changed file instead, and counts as modified unless it carries a status ("A:path" for an
// added file, "R100:old:new" for a rename); the commit message is not read then.
//
// Each line is a `go test -run` pattern, so a selected test can be run as printed:
//
Expand All @@ -27,6 +30,7 @@ package main
import (
"flag"
"fmt"
"maps"
"os"
"path/filepath"
"regexp"
Expand All @@ -39,11 +43,15 @@ import (
func main() {
limit := flag.Int("limit", 50, "maximum number of changed tests to select")
root := flag.String("root", "acceptance", "path to the acceptance directory")
cloud := flag.String("cloud", "", "cloud to list commit message names for: aws, azure, gcp, or empty for a local run")
flag.Parse()

if *limit <= 0 {
fatalf("-limit must be a positive integer, got %d", *limit)
}
if !slices.Contains([]string{"", "aws", "azure", "gcp"}, *cloud) {
fatalf("-cloud must be aws, azure, gcp or empty, got %q", *cloud)
}

dirs, err := selection.FindTestDirs(*root)
if err != nil {
Expand All @@ -55,13 +63,19 @@ func main() {
}

var result selection.Result
if args := flag.Args(); len(args) > 0 {
result = selection.FromDiff(*root, diffFromArgs(args), testDirs, *limit)
var commit selection.CommitSelection
fromGit := len(flag.Args()) == 0
if !fromGit {
result = selection.FromDiff(*root, diffFromArgs(flag.Args()), testDirs, *limit)
} else {
result, err = selection.FromGit(*root, testDirs, *limit)
if err != nil {
fatalf("%s", err)
}
commit, err = selection.FromCommitMessage(*root, testDirs, *cloud)
if err != nil {
fatalf("%s", err)
}
}

fmt.Println(result.Counts())
Expand All @@ -73,6 +87,41 @@ func main() {
if len(result.Selected) == 0 {
fmt.Println(" none of the changed files belong to a test dir")
}

if fromGit {
printNamed(commit)
}
}

// printNamed prints the tests the last commit message names, as go test -run patterns, and
// the warnings the test runner would log for it. A commit that names nothing prints nothing,
// like the test runner, which logs the section only when there is something in it.
func printNamed(commit selection.CommitSelection) {
warnings := commit.Warnings()
if len(commit.Named) == 0 && len(warnings) == 0 {
return
}
fmt.Printf("Selected %d test dirs named in the last commit message (%s)\n", len(commit.Named), commit.Where)
for _, dir := range slices.Sorted(maps.Keys(commit.Named)) {
elements := strings.Split(dir, "/")
var names []string
for _, filters := range commit.Named[dir].Filters {
if filters == nil {
// Every variant runs, so the variant names below add nothing.
names = []string{runPattern(elements)}
break
}
if name := runPattern(append(slices.Clone(elements), filters...)); !slices.Contains(names, name) {
names = append(names, name)
}
}
for _, name := range names {
fmt.Printf(" %s\n", name)
}
}
for _, warning := range warnings {
fmt.Printf("WARNING: %s\n", warning)
}
}

// variantNames returns the go test names of the selected test. go test reads a name as a
Expand Down
119 changes: 93 additions & 26 deletions acceptance/internal/selection/commit.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,25 +35,30 @@ type CommitName struct {
}

// ParseCommitMessage returns the test names on the "<cloud>: <names>" lines of msg. A line may
// list several whitespace-separated names. Lines with any other prefix are ignored, and so are
// lines with a word that is not a go test name: "aws: fix flaky TestAccept retries" is prose,
// and taking its bare TestAccept as a name would run the whole suite.
func ParseCommitMessage(msg string) []CommitName {
var names []CommitName
// list several whitespace-separated names. Lines with any other prefix are ignored. A line with
// a cloud prefix and a word that is not a go test name is returned as rejected instead: "aws:
// fix flaky TestAccept retries" is prose, and taking its bare TestAccept as a name would run
// the whole suite, but "aws: `TestAccept/x`" is a request that must not be dropped silently.
func ParseCommitMessage(msg string) (names []CommitName, rejected []string) {
for line := range strings.SplitSeq(msg, "\n") {
m := commitLineRe.FindStringSubmatch(strings.TrimSpace(line))
line = strings.TrimSpace(line)
m := commitLineRe.FindStringSubmatch(line)
if m == nil {
continue
}
words := strings.Fields(m[2])
if len(words) == 0 || slices.ContainsFunc(words, func(word string) bool { return !strings.HasPrefix(word, "Test") }) {
if len(words) == 0 {
continue
}
if slices.ContainsFunc(words, func(word string) bool { return !strings.HasPrefix(word, "Test") }) {
rejected = append(rejected, line)
continue
}
for _, name := range words {
names = append(names, CommitName{Cloud: m[1], Name: name})
}
}
return names
return names, rejected
}

// NamesForCloud returns the names that apply to cloud (an aws/azure/gcp base cloud name), or
Expand All @@ -71,30 +76,59 @@ func NamesForCloud(names []CommitName, cloud string) []string {
return out
}

// Named maps a test dir to the variant filters of each name that selected it. A nil entry
// means a name selected every variant of the dir. Names are kept apart because each one's
// filters must all hold together, while the names themselves are alternatives.
type Named map[string][][]string
// Named maps a test dir to how the names in a commit message selected it.
type Named map[string]*NamedDir

// NamedDir records the names that selected one test dir.
type NamedDir struct {
// Filters holds the variant filters of each name that selected the dir. A nil entry means
// a name selected every variant of the dir. Names are kept apart because each one's
// filters must all hold together, while the names themselves are alternatives.
Filters [][]string

// Exact is set when a name is the dir itself or one of its variants, as opposed to a
// path above it (or a bare TestAccept) that selects a whole subtree.
Exact bool
}

// Has reports whether any name selected dir.
func (n Named) Has(dir string) bool {
_, ok := n[dir]
return ok
}

// Exact reports whether a name is dir itself or one of its variants. Only such names lift
// CloudSlow: a name like "cloud: TestAccept/bundle" asks for a subtree, not for its slowest
// tests, which -short keeps out of the PR run's time budget.
func (n Named) Exact(dir string) bool {
d, ok := n[dir]
return ok && d.Exact
}

// Matches reports whether a name selected the variant envset of dir.
func (n Named) Matches(dir string, envset []string) bool {
for _, filters := range n[dir] {
d, ok := n[dir]
if !ok {
return false
}
for _, filters := range d.Filters {
if filters == nil || MatchesFilters(envset, filters) {
return true
}
}
return false
}

// add records that a name selected dir with filters, nil for every variant.
func (n Named) add(dir string, filters []string) {
n[dir] = append(n[dir], filters)
// add records that a name selected dir with filters, nil for every variant. exact is set
// when the name is dir or one of its variants.
func (n Named) add(dir string, filters []string, exact bool) {
d, ok := n[dir]
if !ok {
d = &NamedDir{}
n[dir] = d
}
d.Filters = append(d.Filters, filters)
d.Exact = d.Exact || exact
}

// ResolveNames maps go test names to the test dirs they select, and returns the names that
Expand Down Expand Up @@ -126,7 +160,7 @@ func resolveName(root, name string, testDirs map[string]bool, named Named) bool
found := false
for dir := range testDirs {
if rest == "" || dir == rest || strings.HasPrefix(dir, rest+"/") {
named.add(dir, nil)
named.add(dir, nil, rest != "" && dir == rest)
found = true
}
}
Expand All @@ -144,7 +178,7 @@ func resolveName(root, name string, testDirs map[string]bool, named Named) bool
if !slices.ContainsFunc(Variants(root, dir), func(envset []string) bool { return hasAll(envset, filters) }) {
return false
}
named.add(dir, filters)
named.add(dir, filters, true)
return true
}
return false
Expand Down Expand Up @@ -186,22 +220,55 @@ func commitMessage(rev string) (string, error) {
return string(out), nil
}

// CommitSelection is what the last commit message selects for one run.
type CommitSelection struct {
// Named holds the selected test dirs.
Named Named

// Matched lists the names that selected a test, Unmatched the ones that selected none.
Matched []string
Unmatched []string

// Rejected lists the lines with a cloud prefix that were not read as test names (see
// ParseCommitMessage).
Rejected []string

// Where is the cloud the selection is for, or "local".
Where string
}

// Summary is the selection on one line, for the test log.
func (c CommitSelection) Summary() string {
return fmt.Sprintf("Selected %d test dirs named in the last commit message (%s): %s", len(c.Named), c.Where, strings.Join(c.Matched, " "))
}

// Warnings returns one line per name that selected nothing and per rejected line, so a
// request that is not honored is visible in the test log.
func (c CommitSelection) Warnings() []string {
var warnings []string
for _, name := range c.Unmatched {
warnings = append(warnings, fmt.Sprintf("%q in the last commit message matches no acceptance test", name))
}
for _, line := range c.Rejected {
warnings = append(warnings, fmt.Sprintf("ignoring %q in the last commit message: every word after the cloud prefix must be a test name such as TestAccept/<path>", line))
}
return warnings
}

// FromCommitMessage selects among testDirs the tests the last commit message names for cloud
// (empty for a local run, see NamesForCloud). root is the acceptance directory. It returns the
// selection, a one-line summary of the names that matched for the test log, and the names that
// matched no test.
func FromCommitMessage(root string, testDirs map[string]bool, cloud string) (Named, string, []string, error) {
// (empty for a local run, see NamesForCloud). root is the acceptance directory.
func FromCommitMessage(root string, testDirs map[string]bool, cloud string) (CommitSelection, error) {
msg, err := LastCommitMessage()
if err != nil {
return nil, "", nil, err
return CommitSelection{}, err
}
names := NamesForCloud(ParseCommitMessage(msg), cloud)
parsed, rejected := ParseCommitMessage(msg)
names := NamesForCloud(parsed, cloud)
named, unmatched := ResolveNames(root, names, testDirs)
matched := slices.DeleteFunc(names, func(name string) bool { return slices.Contains(unmatched, name) })
where := "local"
if cloud != "" {
where = cloud
}
summary := fmt.Sprintf("Selected %d test dirs named in the last commit message (%s): %s", len(named), where, strings.Join(matched, " "))
return named, summary, unmatched, nil
return CommitSelection{Named: named, Matched: matched, Unmatched: unmatched, Rejected: rejected, Where: where}, nil
}
Loading
Loading