Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/air/empty-snapshot.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* `air run` now refuses to upload a `code_source` snapshot that selects no files (for example when `root_path` is git-ignored) and explains the cause, and `--dry-run` runs the snapshot file selection and reports the file count. ([#7570](https://github.com/databricks/cli/pull/7570))
9 changes: 9 additions & 0 deletions acceptance/air/run/ignored-root.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
experiment_name: smoke-test
command: python main.py
compute:
accelerator_type: GPU_1xH100
num_accelerators: 1
code_source:
type: snapshot
snapshot:
root_path: .scratch/proj
13 changes: 12 additions & 1 deletion acceptance/air/run/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -49,11 +49,22 @@ Exit code: 1
>>> [CLI] air run -f valid.yaml --dry-run --watch
Dry run: configuration for "smoke-test" is valid; not submitting.

=== code_source config passes validation
=== code_source config passes validation and reports the selected files

>>> [CLI] air run -f with-code-source.yaml --dry-run
Dry run: code_source snapshot would include 1 file(s).
Dry run: configuration for "smoke-test" is valid; not submitting.

=== git-ignored root_path is rejected instead of selecting no files

>>> [CLI] air run -f ignored-root.yaml --dry-run
Error: code_source snapshot of [TEST_TMP_DIR]/.scratch/proj contains no files; refusing to upload an empty snapshot: root_path is ignored by git (.gitignore:1:.scratch/), and the snapshot only includes files git does not ignore. Move the project out of the ignored directory, un-ignore it, or change code_source.root_path. Inspect the rule with `git -C [TEST_TMP_DIR]/.scratch/proj check-ignore -v .`

Tip: use the --debug flag to see more details and a trace of this error:
databricks --debug air run …

Exit code: 1

=== git.remote is rejected

>>> [CLI] air run -f git-remote.yaml --dry-run
Expand Down
16 changes: 15 additions & 1 deletion acceptance/air/run/script
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,23 @@ errcode trace $CLI air run -f valid.yaml --dry-run --override compute.num_accele
title "watch is ignored with dry-run (nothing is submitted)"
trace $CLI air run -f valid.yaml --dry-run --watch

title "code_source config passes validation"
title "code_source config passes validation and reports the selected files"
# The dry run resolves git.branch and include_paths locally, so it needs a repo.
git init -qb main
git config user.name "Tester"
git config user.email "tester@databricks.com"
git config core.hooksPath no-hooks
git add src
git commit -qm 'Add src'
trace $CLI air run -f with-code-source.yaml --dry-run

title "git-ignored root_path is rejected instead of selecting no files"
echo '.scratch/' > .gitignore
mkdir -p .scratch/proj
echo 'print(1)' > .scratch/proj/main.py
errcode trace $CLI air run -f ignored-root.yaml --dry-run
rm -fr .git .gitignore .scratch

title "git.remote is rejected"
errcode trace $CLI air run -f git-remote.yaml --dry-run

Expand Down
1 change: 1 addition & 0 deletions acceptance/air/run/src/train.py
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
print("train")
24 changes: 19 additions & 5 deletions cmd/air/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,12 @@ const computeOptionsHelpTimeout = 3 * time.Second

// runResult is the JSON payload for `air run`.
type runResult struct {
Status string `json:"status"`
DryRun bool `json:"dry_run,omitempty"`
RunID string `json:"run_id,omitempty"`
DashboardURL string `json:"dashboard_url,omitempty"`
Status string `json:"status"`
DryRun bool `json:"dry_run,omitempty"`
// SnapshotFileCount is set on a dry run whose config has a code_source snapshot.
SnapshotFileCount *int `json:"snapshot_file_count,omitempty"`
RunID string `json:"run_id,omitempty"`
DashboardURL string `json:"dashboard_url,omitempty"`
}

const experimentalContainersWarning = `+--------------------------------------------------------------------+
Expand Down Expand Up @@ -159,6 +161,15 @@ field, pass its path to -h as a separate argument:
}

if dryRun {
// Local and credential-free, so it runs before workspace validation.
var snapshotFileCount *int
if cfg.CodeSource != nil && cfg.CodeSource.Snapshot != nil {
count, err := countSnapshotFiles(ctx, cfg.CodeSource.Snapshot, file)
if err != nil {
return err
}
snapshotFileCount = &count
}
idempotencyToken := effectiveIdempotencyToken(idempotencyKey, cfg)
ctx, validationErr := validateDryRunInWorkspace(ctx, cmd, args, cfg, idempotencyToken, validationTimeout)
if err := validateIdempotencyToken(idempotencyToken); err != nil {
Expand All @@ -176,10 +187,13 @@ field, pass its path to -h as a separate argument:
return renderError(ctx, cmd, "VALIDATION_UNAVAILABLE", kind, unavailable.retryable, validationErr)
}
if root.OutputType(cmd) == flags.OutputText {
if snapshotFileCount != nil {
cmdio.LogString(ctx, fmt.Sprintf("Dry run: code_source snapshot would include %d file(s).", *snapshotFileCount))
}
cmdio.LogString(ctx, fmt.Sprintf("Dry run: configuration for %q is valid; not submitting.", cfg.ExperimentName))
return nil
}
return renderEnvelope(ctx, runResult{Status: "DRY_RUN_OK", DryRun: true})
return renderEnvelope(ctx, runResult{Status: "DRY_RUN_OK", DryRun: true, SnapshotFileCount: snapshotFileCount})
}

warnExperimentalContainers(ctx, cfg)
Expand Down
35 changes: 35 additions & 0 deletions cmd/air/run_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -284,3 +284,38 @@ func TestRunDryRunValidatesIdempotencyTokenAfterWorkspaceValidation(t *testing.T
runOptions := validations[0].body["run_options"].(map[string]any)
assert.Equal(t, tooLong, runOptions["idempotency_token"])
}

func snapshotConfig(rootPath string) string {
return minimalConfig + "code_source:\n snapshot:\n root_path: " + rootPath + "\n"
}

func TestRunDryRunReportsSnapshotFileCount(t *testing.T) {
srv := dryRunServer(t, validateConfigResponse{}, nil)
project := t.TempDir()
writeRepoFile(t, project, "main.py", "print(1)\n")
writeRepoFile(t, project, "src/model.py", "print(2)\n")

result := runDryRunCmd(t, dryRunValidationTimeout, flags.OutputText, srv.URL, snapshotConfig(project), "--dry-run")
require.NoError(t, result.err)
assert.Contains(t, result.stderr, "code_source snapshot would include 2 file(s)")

result = runDryRunCmd(t, dryRunValidationTimeout, flags.OutputJSON, srv.URL, snapshotConfig(project), "--dry-run")
require.NoError(t, result.err)
var got struct {
Data runResult `json:"data"`
}
require.NoError(t, json.Unmarshal([]byte(result.stdout), &got))
assert.Equal(t, "DRY_RUN_OK", got.Data.Status)
require.NotNil(t, got.Data.SnapshotFileCount)
assert.Equal(t, 2, *got.Data.SnapshotFileCount)
}

func TestRunDryRunRejectsEmptySnapshot(t *testing.T) {
var requests []dryRunRequest
srv := dryRunServer(t, validateConfigResponse{}, &requests)

result := runDryRunCmd(t, dryRunValidationTimeout, flags.OutputText, srv.URL, snapshotConfig(ignoredProject(t)), "--dry-run")
require.Error(t, result.err)
assert.Contains(t, result.err.Error(), "ignored by git")
assert.NotContains(t, result.stderr, "Dry run: configuration")
}
24 changes: 24 additions & 0 deletions cmd/air/snapshot_git.go
Original file line number Diff line number Diff line change
Expand Up @@ -234,6 +234,30 @@ func (g gitRepo) validateIncludePathsExist(ctx context.Context, commitSHA string
return nil
}

// countFilesAtCommit returns how many files `git archive` would package from
// commitSHA, limited to subtreePrefix and includePaths when set.
func (g gitRepo) countFilesAtCommit(ctx context.Context, commitSHA, subtreePrefix string, includePaths []string) (int, error) {
treeish := commitSHA
lsTreeGit := g
if subtreePrefix != "" {
// ls-tree scopes its listing to the working directory, so listing the
// subtree from inside root_path matches nothing. Run from the repo root,
// as createGitArchiveSnapshot does.
repoRoot, err := g.repositoryRoot(ctx)
if err != nil {
return 0, err
}
lsTreeGit = newGitRepo(repoRoot)
treeish = commitSHA + ":" + subtreePrefix
}
args := append([]string{"ls-tree", "-r", "-z", "--name-only", treeish, "--"}, includePaths...)
Comment thread
dimk0 marked this conversation as resolved.
Resolved
out, err := lsTreeGit.runBytes(ctx, args...)
if err != nil {
return 0, err
}
return bytes.Count(out, []byte{0}), nil
}

// shortSHA abbreviates a commit SHA to 8 chars for log/error messages, tolerating
// user-supplied abbreviations shorter than that.
func shortSHA(sha string) string {
Expand Down
27 changes: 27 additions & 0 deletions cmd/air/snapshot_git_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -361,3 +361,30 @@ func TestCaptureDirtyDiff_IncludePaths(t *testing.T) {
}

var fixedNow = time.Date(2026, 7, 10, 12, 0, 0, 0, time.UTC)

func TestCountFilesAtCommit(t *testing.T) {
ctx := t.Context()
repo := newTestRepo(t)
writeRepoFile(t, repo, "README.md", "repo root")
writeRepoFile(t, repo, "subpkg/train.py", "print()")
writeRepoFile(t, repo, "subpkg/nested/util.py", "pass")
sha := commitAll(t, repo, "init")

count, err := newGitRepo(repo).countFilesAtCommit(ctx, sha, "", nil)
require.NoError(t, err)
assert.Equal(t, 3, count)

// root_path below the repo root: git runs from the subfolder, and the count
// must still cover the whole subtree, as `git archive` does.
subGit := newGitRepo(filepath.Join(repo, "subpkg"))
prefix, err := subGit.repoRelativePrefix(ctx)
require.NoError(t, err)

count, err = subGit.countFilesAtCommit(ctx, sha, prefix, nil)
require.NoError(t, err)
assert.Equal(t, 2, count)

count, err = subGit.countFilesAtCommit(ctx, sha, prefix, []string{"nested"})
require.NoError(t, err)
assert.Equal(t, 1, count)
}
23 changes: 23 additions & 0 deletions cmd/air/snapshot_package.go
Original file line number Diff line number Diff line change
Expand Up @@ -190,5 +190,28 @@ func snapshotFiles(ctx context.Context, repoPath string, includePaths []string,
linkTarget: linkTarget,
})
}
if len(files) == 0 {
return nil, emptySnapshotError(ctx, repoPath, includePaths, isGitRepo)
}
return files, nil
}

// emptySnapshotError explains why no file was selected for the snapshot. An empty
// tarball is accepted by the upload but fails on the node, after compute was
// provisioned, so it is rejected here with the most likely cause.
func emptySnapshotError(ctx context.Context, repoPath string, includePaths []string, isGitRepo bool) error {
root := filepath.ToSlash(repoPath)
prefix := fmt.Sprintf("code_source snapshot of %s contains no files; refusing to upload an empty snapshot", root)
if isGitRepo {
// check-ignore -v prints "<source>:<line>:<pattern>\t<path>" and exits non-zero
// when the path is not ignored.
if out, err := newGitRepo(repoPath).run(ctx, "check-ignore", "-v", "."); err == nil {
rule, _, _ := strings.Cut(strings.TrimSpace(out), "\t")
return fmt.Errorf("%s: root_path is ignored by git (%s), and the snapshot only includes files git does not ignore. Move the project out of the ignored directory, un-ignore it, or change code_source.root_path. Inspect the rule with `git -C %s check-ignore -v .`", prefix, rule, root)
}
}
if len(includePaths) > 0 {
return fmt.Errorf("%s: no file under include_paths (%s) is selected. Check that the paths exist under root_path and are not ignored by git (`git check-ignore -v <path>`)", prefix, strings.Join(includePaths, ", "))
}
return fmt.Errorf("%s: the directory is empty or all of its files are excluded by .gitignore rules (`git check-ignore -v <path>` shows the rule for a file). Check code_source.root_path", prefix)
}
39 changes: 39 additions & 0 deletions cmd/air/snapshot_package_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -285,3 +285,42 @@ func TestSnapshotFilesCapturesModeTypeAndSymlinkTarget(t *testing.T) {
assert.NotZero(t, os.FileMode(byName["current"].mode)&os.ModeSymlink)
assert.Equal(t, "run.sh", byName["current"].linkTarget)
}

// ignoredProject returns a project dir that an enclosing repo's .gitignore excludes.
func ignoredProject(t *testing.T) string {
t.Helper()
repo := t.TempDir()
runGit(t, repo, "init", "-q")
writeRepoFile(t, repo, ".gitignore", ".scratch/\n")
writeRepoFile(t, repo, ".scratch/proj/main.py", "print(1)\n")
return filepath.Join(repo, ".scratch", "proj")
}

func TestSnapshotFiles_RejectsGitIgnoredRoot(t *testing.T) {
project := ignoredProject(t)

_, err := snapshotFiles(t.Context(), project, nil, true)
require.Error(t, err)
assert.Contains(t, err.Error(), "contains no files")
assert.Contains(t, err.Error(), "ignored by git")
// The matching rule and the command to inspect it are both named.
assert.Contains(t, err.Error(), ".gitignore:1:.scratch/")
assert.Contains(t, err.Error(), "check-ignore -v")
}

func TestSnapshotFiles_RejectsEmptyDirectory(t *testing.T) {
_, err := snapshotFiles(t.Context(), t.TempDir(), nil, false)
require.Error(t, err)
assert.Contains(t, err.Error(), "contains no files")
assert.NotContains(t, err.Error(), "ignored by git")
}

func TestSnapshotFiles_RejectsIncludePathsMatchingNothing(t *testing.T) {
repo := t.TempDir()
writeRepoFile(t, repo, "a.txt", "1")

_, err := snapshotFiles(t.Context(), repo, []string{"missing"}, false)
require.Error(t, err)
assert.Contains(t, err.Error(), "contains no files")
assert.Contains(t, err.Error(), "include_paths")
}
19 changes: 19 additions & 0 deletions cmd/air/snapshot_upload.go
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,25 @@ func uploadSnapshotSidecars(ctx context.Context, sidecarStore filer.Filer, sidec
return path.Join(sidecarBase, gitStateName), diffPath
}

// countSnapshotFiles runs the snapshot file selection without packaging or
// uploading, so --dry-run fails on the same selection errors as a submission.
func countSnapshotFiles(ctx context.Context, snap *snapshotSourceConfig, configPath string) (int, error) {
repoPath, err := resolveRootPath(ctx, snap.RootPath, filepath.Dir(configPath))
if err != nil {
return 0, err
}
git := newGitRepo(repoPath)
plan, err := resolveSnapshotPlan(ctx, git, snap.Git, snap.IncludePaths)
if err != nil {
return 0, err
}
if plan.mode == modeGitArchive {
return git.countFilesAtCommit(ctx, plan.commitSHA, plan.subtreePrefix, plan.includePaths)
}
files, err := snapshotFiles(ctx, repoPath, plan.includePaths, plan.isGitRepo)
return len(files), err
}

// snapshotTarName resolves the content-addressed upload filename for the snapshot and,
// for plain_tar, the working-tree file listing used to build both the key and the tarball
// (nil for git_archive, which lists nothing locally). The name is <dirName>_<key>.tar.gz,
Expand Down
Loading