Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 8 additions & 7 deletions cmd/air/image.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"os/exec"
"regexp"
"strings"
"sync"

"github.com/databricks/cli/cmd/root"
"github.com/databricks/cli/libs/cmdctx"
Expand All @@ -27,8 +28,8 @@ const (
var (
// DAR uses the OCI repository-component grammar without dots or slashes,
// which are forbidden in Unity Catalog names.
artifactNamePattern = regexp.MustCompile(`^[a-z0-9]+((_|__|-+)[a-z0-9]+)*$`)
imageTagPattern = regexp.MustCompile(`^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$`)
artifactNamePattern = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(`^[a-z0-9]+((_|__|-+)[a-z0-9]+)*$`) })
imageTagPattern = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(`^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$`) })
)

type imagePushOptions struct {
Expand Down Expand Up @@ -240,14 +241,14 @@ func resolveImagePushOptions(
if err != nil {
return resolvedImagePushOptions{}, err
}
if !artifactNamePattern.MatchString(catalog) {
if !artifactNamePattern().MatchString(catalog) {
return resolvedImagePushOptions{}, fmt.Errorf("invalid catalog %q: use lowercase letters and digits separated by one or two underscores or one or more hyphens", catalog)
}
schema, err := promptImagePushValue(ctx, opts.schema, "Unity Catalog schema", "", "--schema")
if err != nil {
return resolvedImagePushOptions{}, err
}
if !artifactNamePattern.MatchString(schema) {
if !artifactNamePattern().MatchString(schema) {
return resolvedImagePushOptions{}, fmt.Errorf("invalid schema %q: use lowercase letters and digits separated by one or two underscores or one or more hyphens", schema)
}

Expand Down Expand Up @@ -312,7 +313,7 @@ func resolveArtifactAndTag(ctx context.Context, image, defaultArtifact, defaultT
tag = after
}

if !artifactNamePattern.MatchString(artifact) {
if !artifactNamePattern().MatchString(artifact) {
return "", "", fmt.Errorf("invalid artifact name %q: use lowercase letters and digits separated by one or two underscores or one or more hyphens", artifact)
}
if tag == "" {
Expand All @@ -321,7 +322,7 @@ func resolveArtifactAndTag(ctx context.Context, image, defaultArtifact, defaultT
if len(tag) > maxImageTagLength {
return "", "", fmt.Errorf("invalid image tag: must be %d characters or less", maxImageTagLength)
}
if !imageTagPattern.MatchString(tag) {
if !imageTagPattern().MatchString(tag) {
return "", "", fmt.Errorf("invalid image tag %q", tag)
}
return artifact, tag, nil
Expand All @@ -338,7 +339,7 @@ func sourceImageDefaults(source string) (string, string) {
tag = source[lastColon+1:]
}
artifact := repository[strings.LastIndex(repository, "/")+1:]
if !artifactNamePattern.MatchString(artifact) {
if !artifactNamePattern().MatchString(artifact) {
artifact = ""
}
return artifact, tag
Expand Down
81 changes: 43 additions & 38 deletions cmd/air/logdetect.go
Original file line number Diff line number Diff line change
@@ -1,50 +1,55 @@
package aircmd

import "regexp"
import (
"regexp"
"sync"
)

// fatalPatterns match log lines that signal a run-ending failure (OOM, NCCL
// timeouts, CUDA errors, segfaults, etc.). In --json mode a matching line emits
// an ALERT event alongside its LOG event, giving an agent an immediate signal.
var fatalPatterns = []*regexp.Regexp{
// OOM
regexp.MustCompile(`(?i)CUDA out of memory`),
regexp.MustCompile(`(?i)Out of memory: Kill(ed)? process`),
// Signals
regexp.MustCompile(`(?i)signal\s+(9|SIGKILL|SIGTERM)`),
// NCCL / collective
regexp.MustCompile(`Watchdog caught collective operation timeout`),
regexp.MustCompile(`(?i)NCCL WARN .*(Conn|Net|IB|timeout|unhandled)`),
regexp.MustCompile(`Got async error event`),
regexp.MustCompile(`transport/net_ib\.cc:\d+.*WARN`),
// CUDA
regexp.MustCompile(`(?i)CUDA(?: runtime)? error`),
regexp.MustCompile(`(?i)an illegal memory access was encountered`),
regexp.MustCompile(`CUDA kernel errors might be asynchronously reported`),
// Segfault
regexp.MustCompile(`(?i)segmentation fault`),
// Composer / llmfoundry specific
regexp.MustCompile(`composer\.utils\..*Error`),
regexp.MustCompile(`(?i)composer.*OutOfMemory`),
// torch.distributed
regexp.MustCompile(`torch\.distributed\..*(?:Error|Exception)`),
regexp.MustCompile(`(?i)(?:TCPStore|Store).*timed?\s*out`),
// GPU hardware (Xid from dmesg / driver)
regexp.MustCompile(`Xid.*\b(48|63|64|79|94|95)\b`),
// Streaming dataset
regexp.MustCompile(`streaming\.base\..*(?:Error|Exception)`),
// Bare "Killed" on its own line means OOM-killer or similar
regexp.MustCompile(`^\s*Killed\s*$`),
// MLflow stall — training has stopped logging metrics
regexp.MustCompile(`\[MLflow Logger\]\[Warning\] No new logs have been emitted`),
// The launch script prints this when the user's command fails; [1-9]\d* skips exit code 0.
regexp.MustCompile(`ERROR: Script failed with exit code [1-9]\d* after \d+s`),
// Missing command (exit 127), e.g. a typo.
regexp.MustCompile(`(?i)command not found`),
}
var fatalPatterns = sync.OnceValue(func() []*regexp.Regexp {
return []*regexp.Regexp{
// OOM
regexp.MustCompile(`(?i)CUDA out of memory`),
regexp.MustCompile(`(?i)Out of memory: Kill(ed)? process`),
// Signals
regexp.MustCompile(`(?i)signal\s+(9|SIGKILL|SIGTERM)`),
// NCCL / collective
regexp.MustCompile(`Watchdog caught collective operation timeout`),
regexp.MustCompile(`(?i)NCCL WARN .*(Conn|Net|IB|timeout|unhandled)`),
regexp.MustCompile(`Got async error event`),
regexp.MustCompile(`transport/net_ib\.cc:\d+.*WARN`),
// CUDA
regexp.MustCompile(`(?i)CUDA(?: runtime)? error`),
regexp.MustCompile(`(?i)an illegal memory access was encountered`),
regexp.MustCompile(`CUDA kernel errors might be asynchronously reported`),
// Segfault
regexp.MustCompile(`(?i)segmentation fault`),
// Composer / llmfoundry specific
regexp.MustCompile(`composer\.utils\..*Error`),
regexp.MustCompile(`(?i)composer.*OutOfMemory`),
// torch.distributed
regexp.MustCompile(`torch\.distributed\..*(?:Error|Exception)`),
regexp.MustCompile(`(?i)(?:TCPStore|Store).*timed?\s*out`),
// GPU hardware (Xid from dmesg / driver)
regexp.MustCompile(`Xid.*\b(48|63|64|79|94|95)\b`),
// Streaming dataset
regexp.MustCompile(`streaming\.base\..*(?:Error|Exception)`),
// Bare "Killed" on its own line means OOM-killer or similar
regexp.MustCompile(`^\s*Killed\s*$`),
// MLflow stall — training has stopped logging metrics
regexp.MustCompile(`\[MLflow Logger\]\[Warning\] No new logs have been emitted`),
// The launch script prints this when the user's command fails; [1-9]\d* skips exit code 0.
regexp.MustCompile(`ERROR: Script failed with exit code [1-9]\d* after \d+s`),
// Missing command (exit 127), e.g. a typo.
regexp.MustCompile(`(?i)command not found`),
}
})

// matchFatalPattern reports whether a log line matches a fatal-failure pattern.
func matchFatalPattern(line string) bool {
for _, p := range fatalPatterns {
for _, p := range fatalPatterns() {
if p.MatchString(line) {
return true
}
Expand Down
8 changes: 4 additions & 4 deletions cmd/air/logmlflow.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,12 +58,12 @@ func volumeArtifactRoot(root string) (string, bool) {

// chunkFilePattern matches a log chunk file (logs-<index>.chunk.txt); group 1 is
// the chunk index. The sidecar splits stdout into 4MB chunks, index ascending.
var chunkFilePattern = regexp.MustCompile(`^logs-(\d+)\.chunk\.txt$`)
var chunkFilePattern = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(`^logs-(\d+)\.chunk\.txt$`) })

// oldFormatNodeDir matches a bare per-node log dir (logs/node_<n>). The
// attempt-prefixed layout nests these under logs/attempt_<n>/, so a bare
// logs/node_<n> only appears in the old layout.
var oldFormatNodeDir = regexp.MustCompile(`^logs/node_\d+$`)
var oldFormatNodeDir = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(`^logs/node_\d+$`) })

// artifactDownloadClient fetches pre-signed artifact URLs with connect and
// response-header timeouts, so a stalled storage backend can't hang the command.
Expand Down Expand Up @@ -268,7 +268,7 @@ func discoverAttemptPrefix(ctx context.Context, w *databricks.WorkspaceClient, m

attemptDir := fmt.Sprintf("logs/attempt_%d", attempt)
for _, f := range files {
if oldFormatNodeDir.MatchString(f.Path) {
if oldFormatNodeDir().MatchString(f.Path) {
return false, nil
}
if f.Path == attemptDir {
Expand Down Expand Up @@ -307,7 +307,7 @@ func listLogChunks(ctx context.Context, w *databricks.WorkspaceClient, mlflowRun
var chunks []logChunk
for _, f := range files {
base := path.Base(f.Path)
m := chunkFilePattern.FindStringSubmatch(base)
m := chunkFilePattern().FindStringSubmatch(base)
if m == nil {
continue
}
Expand Down
6 changes: 3 additions & 3 deletions cmd/air/logmlflow_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,12 +57,12 @@ func TestChunkFileName(t *testing.T) {
}

func TestChunkFilePattern(t *testing.T) {
m := chunkFilePattern.FindStringSubmatch("logs-12.chunk.txt")
m := chunkFilePattern().FindStringSubmatch("logs-12.chunk.txt")
require.NotNil(t, m)
assert.Equal(t, "12", m[1])

assert.Nil(t, chunkFilePattern.FindStringSubmatch("logs-12.chunk.txt.bak"))
assert.Nil(t, chunkFilePattern.FindStringSubmatch("node_0"))
assert.Nil(t, chunkFilePattern().FindStringSubmatch("logs-12.chunk.txt.bak"))
assert.Nil(t, chunkFilePattern().FindStringSubmatch("node_0"))
}

// artifactListServer serves a fixed artifacts/list response for any path.
Expand Down
31 changes: 17 additions & 14 deletions cmd/air/runconfig.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"regexp"
"slices"
"strings"
"sync"

"go.yaml.in/yaml/v3"
)
Expand All @@ -27,25 +28,27 @@ import (
// REGEX_TASK_KEY_CHARS: ASCII alphanumeric, hyphen, underscore only (no periods).
// Explicit ASCII class, not \w: \w matches Unicode letters that the ASCII-only
// Jobs API task_key rejects.
var taskKeyRe = regexp.MustCompile(`^[A-Za-z0-9_-]+$`)
var taskKeyRe = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(`^[A-Za-z0-9_-]+$`) })

// gitRefRe guards the branch name against command injection (it flows into git
// exec args). Only safe ref characters are allowed.
var gitRefRe = regexp.MustCompile(`^[\w./-]+$`)
var gitRefRe = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(`^[\w./-]+$`) })

// Canonical UUID (8-4-4-4-12 hex). Usage policy ids are server-generated UUIDs,
// so an obviously-wrong value (e.g. a policy name pasted into usage_policy_id)
// is rejected up front with a hint pointing at usage_policy_name.
var uuidRe = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`)
var uuidRe = sync.OnceValue(func() *regexp.Regexp {
return regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`)
})

// Unity Catalog image paths are submitted as <catalog>.<schema>.<image>:<tag>.
// Keep validation structural only here: object existence and permissions are
// workspace checks performed by the backend.
var unityCatalogImageRe = regexp.MustCompile(`^[^.:\s]+\.[^.:\s]+\.[^.:\s]+:[^:\s]+$`)
var unityCatalogImageRe = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(`^[^.:\s]+\.[^.:\s]+\.[^.:\s]+:[^:\s]+$`) })

var containerNameRe = regexp.MustCompile(`^[a-z]([-a-z0-9]{0,30}[a-z0-9])?$`)
var containerNameRe = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(`^[a-z]([-a-z0-9]{0,30}[a-z0-9])?$`) })

var secretTemplateRe = regexp.MustCompile(`^\{\{secrets/([^/{}]+)/([^/{}]+)\}\}$`)
var secretTemplateRe = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(`^\{\{secrets/([^/{}]+)/([^/{}]+)\}\}$`) })

// runConfig is the top-level run YAML schema: experiment_name + compute /
// environment / code_source plus the command and run options.
Expand Down Expand Up @@ -159,7 +162,7 @@ func (c *runConfig) validate() error {
if len(v) > 100 {
return fmt.Errorf("mlflow_run_name must be 100 characters or less (got %d)", len(v))
}
if !taskKeyRe.MatchString(v) {
if !taskKeyRe().MatchString(v) {
return fmt.Errorf("invalid mlflow_run_name %q: only alphanumeric characters, hyphens, and underscores are allowed", v)
}
}
Expand Down Expand Up @@ -215,7 +218,7 @@ func (c *runConfig) validate() error {
if v == "" {
return errors.New("usage_policy_id must not be empty")
}
if !uuidRe.MatchString(v) {
if !uuidRe().MatchString(v) {
return fmt.Errorf("usage_policy_id must be a UUID (for example, '12345678-90ab-cdef-1234-567890abcdef'), got: %s. To assign a policy by name instead, use usage_policy_name", v)
}
}
Expand Down Expand Up @@ -256,7 +259,7 @@ func (c *runConfig) validateContainers() error {
for i := range c.Containers {
container := &c.Containers[i]
prefix := fmt.Sprintf("containers[%d]", i)
if !containerNameRe.MatchString(container.Name) {
if !containerNameRe().MatchString(container.Name) {
return fmt.Errorf("%s.name must start with a lowercase letter, contain only lowercase letters, digits, or '-', end with a letter or digit, and be at most 32 characters, got %q", prefix, container.Name)
}
if _, exists := names[container.Name]; exists {
Expand All @@ -272,15 +275,15 @@ func (c *runConfig) validateContainers() error {
if len(container.Ranks) == 0 {
return fmt.Errorf("%s.ranks must contain at least one rank", prefix)
}
if !unityCatalogImageRe.MatchString(container.UnityCatalogImage) {
if !unityCatalogImageRe().MatchString(container.UnityCatalogImage) {
return fmt.Errorf("%s.unity_catalog_image must be in the format '<catalog>.<schema>.<image>:<tag>', got %q", prefix, container.UnityCatalogImage)
}
if container.EnvironmentVariables != nil {
if len(container.EnvironmentVariables.Files) > 0 {
return fmt.Errorf("%s.environment_variables.files is not supported while container environment variables use workspace sidecars; use environment_variables.variables", prefix)
}
for name, value := range container.EnvironmentVariables.Variables {
if strings.HasPrefix(value, "{{secrets/") && !secretTemplateRe.MatchString(value) {
if strings.HasPrefix(value, "{{secrets/") && !secretTemplateRe().MatchString(value) {
return fmt.Errorf("%s.environment_variables.variables.%s has an invalid secret reference %q; expected {{secrets/<scope>/<key>}}", prefix, name, value)
}
}
Expand Down Expand Up @@ -313,7 +316,7 @@ func validateExperimentName(v string) error {
if len(v) > 100 {
return fmt.Errorf("experiment_name must be 100 characters or less (got %d); this is the Jobs API task_key length limit", len(v))
}
if !taskKeyRe.MatchString(v) {
if !taskKeyRe().MatchString(v) {
return fmt.Errorf("invalid experiment_name %q: only alphanumeric characters, hyphens (-), and underscores (_) are allowed", v)
}
return nil
Expand Down Expand Up @@ -355,7 +358,7 @@ type environmentConfig struct {

func (e *environmentConfig) validate() error {
unityCatalogImage := e.UnityCatalogImage
if unityCatalogImage != "" && !unityCatalogImageRe.MatchString(unityCatalogImage) {
if unityCatalogImage != "" && !unityCatalogImageRe().MatchString(unityCatalogImage) {
return fmt.Errorf("environment.unity_catalog_image must be in the format '<catalog>.<schema>.<image>:<tag>', got %q", e.UnityCatalogImage)
}

Expand Down Expand Up @@ -505,7 +508,7 @@ type gitRef struct {
}

func (g *gitRef) validate() error {
if g.Branch != nil && !gitRefRe.MatchString(*g.Branch) {
if g.Branch != nil && !gitRefRe().MatchString(*g.Branch) {
return fmt.Errorf("invalid git.branch format %q: only alphanumeric characters, hyphens, dots, slashes, and underscores are allowed", *g.Branch)
}

Expand Down
5 changes: 3 additions & 2 deletions cmd/air/runconfig_load.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"regexp"
"strconv"
"strings"
"sync"

"go.yaml.in/yaml/v3"
)
Expand Down Expand Up @@ -142,7 +143,7 @@ func stripYAMLComments(node *yaml.Node) {
}
}

var runtimeVersionRe = regexp.MustCompile(`^[0-9]+$`)
var runtimeVersionRe = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(`^[0-9]+$`) })

const databricksAIPrefix = "databricks_ai_v"

Expand All @@ -153,7 +154,7 @@ func validateRuntimeVersion(version, source string) (string, error) {
if usesDatabricksAI {
numeric = strings.TrimPrefix(normalized, databricksAIPrefix)
}
if !runtimeVersionRe.MatchString(numeric) {
if !runtimeVersionRe().MatchString(numeric) {
return "", fmt.Errorf("unsupported client image version %q in %s: version must be an integer, optionally prefixed with databricks_ai_v", version, source)
}
if !usesDatabricksAI {
Expand Down
2 changes: 1 addition & 1 deletion cmd/air/runupload.go
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ func containerEnvironmentEntries(cfg *runConfig, container containerConfig) (map
for name, value := range container.EnvironmentVariables.Variables {
delete(plain, name)
delete(secrets, name)
match := secretTemplateRe.FindStringSubmatch(value)
match := secretTemplateRe().FindStringSubmatch(value)
if match != nil {
secrets[name] = match[1] + "/" + match[2]
} else {
Expand Down
Loading