diff --git a/.nextchanges/air/air-code-source-without-scm.md b/.nextchanges/air/air-code-source-without-scm.md new file mode 100644 index 00000000000..72cc4993b69 --- /dev/null +++ b/.nextchanges/air/air-code-source-without-scm.md @@ -0,0 +1 @@ +* Package `code_source` directories outside a git repository without a `git` binary on PATH, and fail early with a clear error when the directory is inside a repository or `code_source.git` is set. ([#7625](https://github.com/databricks/cli/pull/7625)) diff --git a/cmd/air/run.go b/cmd/air/run.go index 20a7683461f..33d15f4f290 100644 --- a/cmd/air/run.go +++ b/cmd/air/run.go @@ -7,6 +7,7 @@ import ( "io" "os" "os/signal" + "path/filepath" "strconv" "strings" "time" @@ -21,6 +22,20 @@ import ( "github.com/spf13/cobra" ) +// checkCodeSourceGit fails before the config is validated or anything is packaged when +// git is missing and the code_source cannot be packaged without it. Runs for dry runs too. +func checkCodeSourceGit(ctx context.Context, cfg *runConfig, configPath string) error { + if cfg.CodeSource == nil || cfg.CodeSource.Snapshot == nil || gitInstalled() { + return nil + } + snap := cfg.CodeSource.Snapshot + root, err := resolveRootPath(ctx, snap.RootPath, filepath.Dir(configPath)) + if err != nil { + return err + } + return gitRequiredError(root, snap.Git) +} + // dryRunValidationTimeout bounds only the config:validate request. const dryRunValidationTimeout = 15 * time.Second @@ -159,6 +174,9 @@ field, pass its path to -h as a separate argument: if err != nil { return err } + if err := checkCodeSourceGit(ctx, cfg, file); err != nil { + return err + } if dryRun { // Local and credential-free, so it runs before workspace validation. diff --git a/cmd/air/run_test.go b/cmd/air/run_test.go index cedae60e672..1f8e8455b2b 100644 --- a/cmd/air/run_test.go +++ b/cmd/air/run_test.go @@ -5,6 +5,9 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "os" + "os/exec" + "path/filepath" "strings" "sync/atomic" "testing" @@ -87,6 +90,40 @@ func TestRunSubmitTextOutput(t *testing.T) { assert.Zero(t, counts.runGetOutput.Load(), "bare submission must not poll runs/get-output") } +func TestCheckCodeSourceGit(t *testing.T) { + repo := newTestRepo(t) + sub := filepath.Join(repo, "sub") + require.NoError(t, os.MkdirAll(sub, 0o755)) + plain := t.TempDir() + withCodeSource := func(root string, ref *gitRef) *runConfig { + return &runConfig{CodeSource: &codeSourceConfig{Snapshot: &snapshotSourceConfig{RootPath: root, Git: ref}}} + } + withoutGit(t) + + err := checkCodeSourceGit(t.Context(), withCodeSource(".", nil), filepath.Join(sub, "workload.yaml")) + require.Error(t, err) + assert.Contains(t, err.Error(), "is in a git repository") + assert.Contains(t, err.Error(), "needs git on PATH") + assert.NotContains(t, err.Error(), "exec:") + + err = checkCodeSourceGit(t.Context(), withCodeSource(".", &gitRef{Branch: new("main")}), filepath.Join(plain, "workload.yaml")) + require.Error(t, err) + assert.Contains(t, err.Error(), "code_source.git is set") + + require.NoError(t, checkCodeSourceGit(t.Context(), withCodeSource(".", nil), filepath.Join(plain, "workload.yaml"))) + + // Without code_source the git binary is never looked up, and its absence is not an error. + looked := false + orig := gitLookPath + gitLookPath = func(string) (string, error) { + looked = true + return "", exec.ErrNotFound + } + t.Cleanup(func() { gitLookPath = orig }) + require.NoError(t, checkCodeSourceGit(t.Context(), &runConfig{}, filepath.Join(plain, "workload.yaml"))) + assert.False(t, looked) +} + func TestWarnExperimentalContainers(t *testing.T) { var stdout, stderr bytes.Buffer ctx := cmdio.InContext(t.Context(), cmdio.NewIO(t.Context(), flags.OutputText, nil, &stdout, &stderr, "", "")) diff --git a/cmd/air/snapshot_git.go b/cmd/air/snapshot_git.go index 53d838ef9c0..790bd9f7442 100644 --- a/cmd/air/snapshot_git.go +++ b/cmd/air/snapshot_git.go @@ -7,7 +7,9 @@ import ( "errors" "fmt" "io" + "os" "os/exec" + "path/filepath" "strings" "time" ) @@ -73,6 +75,42 @@ func (g gitRepo) isRepository(ctx context.Context) bool { return strings.TrimSpace(out) == "true" } +// hasGitMetadata reports whether dir holds a repository's `.git` entry: a gitfile, or a +// directory with HEAD. An empty `.git` directory is not a repository, as in git. +func hasGitMetadata(dir string) bool { + info, err := os.Stat(filepath.Join(dir, ".git")) + if err != nil { + return false + } + if info.Mode().IsRegular() { + return true + } + if !info.IsDir() { + return false + } + _, err = os.Stat(filepath.Join(dir, ".git", "HEAD")) + return err == nil +} + +// insideGitRepository is the git-free stand-in for isRepository, used only when no git +// binary is available: repository metadata in dir or one of its ancestors marks a work tree. +func insideGitRepository(dir string) bool { + abs, err := filepath.Abs(dir) + if err != nil { + return false + } + for { + if hasGitMetadata(abs) { + return true + } + parent := filepath.Dir(abs) + if parent == abs { + return false + } + abs = parent + } +} + // repoRelativePrefix returns the path from the repository root to g.path. Git // emits a trailing slash for subdirectories and an empty string at the root. func (g gitRepo) repoRelativePrefix(ctx context.Context) (string, error) { diff --git a/cmd/air/snapshot_package.go b/cmd/air/snapshot_package.go index c1dc9c5f89d..fe20c21447e 100644 --- a/cmd/air/snapshot_package.go +++ b/cmd/air/snapshot_package.go @@ -6,12 +6,17 @@ import ( "errors" "fmt" "io" + "io/fs" "os" "os/exec" + "path" "path/filepath" + "slices" "strings" + libgit "github.com/databricks/cli/libs/git" "github.com/databricks/cli/libs/tarpack" + "github.com/databricks/cli/libs/vfs" "github.com/klauspost/pgzip" ) @@ -135,35 +140,20 @@ type snapshotFile struct { } func snapshotFiles(ctx context.Context, repoPath string, includePaths []string, isGitRepo bool) ([]snapshotFile, error) { - args := []string{"-C", repoPath, "ls-files", "-z", "--cached", "--others", "--exclude-standard"} - if !isGitRepo { - gitDir, err := os.MkdirTemp("", "air-snapshot-git-") - if err != nil { - return nil, fmt.Errorf("failed to create temporary git metadata: %w", err) - } - defer os.RemoveAll(gitDir) - cmd := exec.CommandContext(ctx, "git", "--git-dir", gitDir, "--work-tree", repoPath, "init", "--quiet") - if output, err := cmd.CombinedOutput(); err != nil { - return nil, fmt.Errorf("failed to initialize temporary git metadata: %w: %s", err, strings.TrimSpace(string(output))) - } - args = []string{"--git-dir", gitDir, "--work-tree", repoPath, "ls-files", "-z", "--cached", "--others", "--exclude-standard"} + var names []string + var err error + if !isGitRepo && !gitInstalled() { + names, err = listFilesWithoutGit(ctx, repoPath, includePaths) + } else { + names, err = listFilesWithGit(ctx, repoPath, includePaths, isGitRepo) } - - if len(includePaths) > 0 { - args = append(args, "--") - args = append(args, includePaths...) - } - output, err := exec.CommandContext(ctx, "git", args...).Output() if err != nil { - return nil, fmt.Errorf("failed to evaluate git ignore rules: %w", err) + return nil, err } var files []snapshotFile - for raw := range bytes.SplitSeq(output, []byte{0}) { - if len(raw) == 0 { - continue - } - name := filepath.ToSlash(string(raw)) + for _, raw := range names { + name := filepath.ToSlash(raw) base := filepath.Base(name) if name == ".git" || strings.HasPrefix(name, ".git/") || strings.HasPrefix(base, "._") { continue @@ -196,6 +186,145 @@ func snapshotFiles(ctx context.Context, repoPath string, includePaths []string, return files, nil } +// listFilesWithGit returns the repo-relative paths `git ls-files` selects for repoPath. +// A directory outside any repository gets a temporary git dir so the same ignore +// rules apply without touching the directory. +func listFilesWithGit(ctx context.Context, repoPath string, includePaths []string, isGitRepo bool) ([]string, error) { + args := []string{"-C", repoPath, "ls-files", "-z", "--cached", "--others", "--exclude-standard"} + if !isGitRepo { + gitDir, err := os.MkdirTemp("", "air-snapshot-git-") + if err != nil { + return nil, fmt.Errorf("failed to create temporary git metadata: %w", err) + } + defer os.RemoveAll(gitDir) + cmd := exec.CommandContext(ctx, "git", "--git-dir", gitDir, "--work-tree", repoPath, "init", "--quiet") + if output, err := cmd.CombinedOutput(); err != nil { + return nil, fmt.Errorf("failed to initialize temporary git metadata: %w: %s", err, strings.TrimSpace(string(output))) + } + args = []string{"--git-dir", gitDir, "--work-tree", repoPath, "ls-files", "-z", "--cached", "--others", "--exclude-standard"} + } + + if len(includePaths) > 0 { + args = append(args, "--") + args = append(args, includePaths...) + } + output, err := exec.CommandContext(ctx, "git", args...).Output() + if err != nil { + return nil, fmt.Errorf("failed to evaluate git ignore rules: %w", err) + } + + var names []string + for raw := range bytes.SplitSeq(output, []byte{0}) { + if len(raw) > 0 { + names = append(names, string(raw)) + } + } + return names, nil +} + +// gitLookPath finds the git binary. Tests replace it to simulate a host without git. +var gitLookPath = exec.LookPath + +// gitInstalled reports whether a git binary is on PATH. +func gitInstalled() bool { + _, err := gitLookPath("git") + return err == nil +} + +// listFilesWithoutGit is the fallback for a directory outside any repository when git +// is not on PATH. It walks the tree applying libs/git's ignore rules (the root and +// nested .gitignore files, info/exclude and core.excludesFile from config), lists +// regular files and symlinks without following symlinks, and emits a nested repository +// as one "dir/" entry, as git does. +// +// Known differences from `git ls-files --others --exclude-standard`, which only matter +// on hosts without git (with git on PATH the git listing is used unchanged): +// - Rule precedence: libs/git returns the first matching rule from the root down, so +// a root "*.log" excludes nested/keep.log even when nested/.gitignore has "!keep.log". +// - include_paths use path.Match, where "*" does not cross "/"; git's pathspecs do. +func listFilesWithoutGit(ctx context.Context, repoPath string, includePaths []string) ([]string, error) { + repo, err := libgit.NewRepository(ctx, vfs.MustNew(repoPath)) + if err != nil { + return nil, fmt.Errorf("failed to read .gitignore rules: %w", err) + } + + var names []string + err = filepath.WalkDir(repoPath, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if p == repoPath { + return nil + } + rel, err := filepath.Rel(repoPath, p) + if err != nil { + return err + } + name := filepath.ToSlash(rel) + if d.Name() == ".git" { + if d.IsDir() { + return fs.SkipDir + } + return nil + } + if d.IsDir() { + ignored, err := repo.Ignore(name + "/") + if err != nil { + return err + } + if ignored { + return fs.SkipDir + } + if hasGitMetadata(p) { + names = append(names, name+"/") + return fs.SkipDir + } + return nil + } + ignored, err := repo.Ignore(name) + if err != nil { + return err + } + if !ignored { + names = append(names, name) + } + return nil + }) + if err != nil { + return nil, fmt.Errorf("failed to list files: %w", err) + } + + slices.Sort(names) + if len(includePaths) > 0 { + names = slices.DeleteFunc(names, func(name string) bool { + return !matchesIncludePath(name, includePaths) + }) + } + return names, nil +} + +// matchesIncludePath reports whether name, or a directory containing it, matches one of +// the include_paths. +func matchesIncludePath(name string, includePaths []string) bool { + for _, spec := range includePaths { + spec = path.Clean(filepath.ToSlash(strings.TrimSpace(spec))) + if spec == "." { + return true + } + for p := strings.TrimSuffix(name, "/"); p != ""; { + if ok, _ := path.Match(spec, p); ok { + return true + } + i := strings.LastIndex(p, "/") + if i < 0 { + break + } + p = p[:i] + } + } + return false +} + // emptySnapshotError explains why no file was selected for the snapshot. An empty // tarball is accepted by the upload but fails on the node, after compute was // provisioned, so it is rejected here with the most likely cause. diff --git a/cmd/air/snapshot_package_test.go b/cmd/air/snapshot_package_test.go index 068997e2671..4d92f4d336a 100644 --- a/cmd/air/snapshot_package_test.go +++ b/cmd/air/snapshot_package_test.go @@ -5,6 +5,7 @@ import ( "compress/gzip" "context" "os" + "os/exec" "path/filepath" "slices" "strings" @@ -315,6 +316,135 @@ func TestSnapshotFiles_RejectsEmptyDirectory(t *testing.T) { assert.NotContains(t, err.Error(), "ignored by git") } +// withoutGit makes the git binary look absent for the rest of the test. +func withoutGit(t *testing.T) { + t.Helper() + orig := gitLookPath + gitLookPath = func(string) (string, error) { return "", exec.ErrNotFound } + t.Cleanup(func() { gitLookPath = orig }) +} + +// writeFallbackFixture lays out a directory outside any repository that covers the cases +// the git-free listing must select as git does: root and nested .gitignore files, +// negation, an ignored directory, dotfiles, and symlinks that are listed but not followed. +func writeFallbackFixture(t *testing.T) string { + t.Helper() + dir := t.TempDir() + writeRepoFile(t, dir, ".gitignore", "build/\n*.log\n") + writeRepoFile(t, dir, "keep.txt", "1") + writeRepoFile(t, dir, "drop.log", "1") + writeRepoFile(t, dir, "build/out.py", "1") + writeRepoFile(t, dir, "README.md", "1") + writeRepoFile(t, dir, ".dotfile", "1") + writeRepoFile(t, dir, ".hidden/secret.txt", "1") + writeRepoFile(t, dir, "a/x.py", "1") + writeRepoFile(t, dir, "src/m.py", "1") + writeRepoFile(t, dir, "src/deep/n.py", "1") + writeRepoFile(t, dir, "nested/.gitignore", "*.tmp\n!keep.tmp\n") + writeRepoFile(t, dir, "nested/drop.tmp", "1") + writeRepoFile(t, dir, "nested/keep.tmp", "1") + writeRepoFile(t, dir, "nested/inner/z.py", "1") + require.NoError(t, os.Symlink("src/m.py", filepath.Join(dir, "lnk.py"))) + require.NoError(t, os.Symlink("src", filepath.Join(dir, "symdir"))) + return dir +} + +// selectedNames returns the repo-relative paths snapshotFiles selects, sorted. +func selectedNames(t *testing.T, dir string, includePaths []string, isGitRepo bool) []string { + t.Helper() + files, err := snapshotFiles(t.Context(), dir, includePaths, isGitRepo) + require.NoError(t, err) + names := make([]string, 0, len(files)) + for _, f := range files { + names = append(names, filepath.ToSlash(f.rel)) + } + slices.Sort(names) + return names +} + +// The git listing and the git-free fallback select the same files for the common cases, +// and the git listing is pinned here so changing the fallback cannot change it. +func TestSnapshotFilesWithoutGit_MatchesGitListing(t *testing.T) { + cases := []struct { + name string + includePaths []string + want []string + }{ + { + name: "whole tree", + want: []string{ + ".dotfile", ".gitignore", ".hidden/secret.txt", "README.md", "a/x.py", + "keep.txt", "lnk.py", "nested/.gitignore", "nested/inner/z.py", + "nested/keep.tmp", "src/deep/n.py", "src/m.py", "symdir", + }, + }, + { + name: "directory include_path", + includePaths: []string{"src"}, + want: []string{"src/deep/n.py", "src/m.py"}, + }, + { + name: "file include_path", + includePaths: []string{"a/x.py"}, + want: []string{"a/x.py"}, + }, + { + name: "nested include_path with nested rules", + includePaths: []string{"nested"}, + want: []string{"nested/.gitignore", "nested/inner/z.py", "nested/keep.tmp"}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + dir := writeFallbackFixture(t) + assert.Equal(t, tc.want, selectedNames(t, dir, tc.includePaths, false), "git listing changed") + + withoutGit(t) + assert.Equal(t, tc.want, selectedNames(t, dir, tc.includePaths, false), "git-free fallback") + }) + } +} + +// A nested repository is one "dir/" entry in both listings, and its contents are not walked. +func TestSnapshotFilesWithoutGit_NestedRepoIsOneEntry(t *testing.T) { + dir := t.TempDir() + writeRepoFile(t, dir, "top.py", "1") + nested := filepath.Join(dir, "nested") + require.NoError(t, os.MkdirAll(nested, 0o755)) + runGit(t, nested, "init", "-q") + writeRepoFile(t, nested, "inner.py", "1") + + want := []string{"nested/", "top.py"} + assert.Equal(t, want, selectedNames(t, dir, nil, false)) + withoutGit(t) + assert.Equal(t, want, selectedNames(t, dir, nil, false)) +} + +// Known difference: a root *.log excludes nested/keep.log even though the nested file +// re-includes it, because libs/git returns the first matching rule from the root down. +func TestSnapshotFilesWithoutGit_NestedNegationDiffersFromGit(t *testing.T) { + dir := t.TempDir() + writeRepoFile(t, dir, ".gitignore", "*.log\n") + writeRepoFile(t, dir, "nested/.gitignore", "!keep.log\n") + writeRepoFile(t, dir, "nested/keep.log", "1") + + assert.Contains(t, selectedNames(t, dir, nil, false), "nested/keep.log") + withoutGit(t) + assert.NotContains(t, selectedNames(t, dir, nil, false), "nested/keep.log") +} + +// Known difference: include_paths globs use path.Match, where "*" does not cross "/", +// so "*.py" selects only the root file (git's pathspec selects a/x.py too). +func TestSnapshotFilesWithoutGit_IncludeGlobDiffersFromGit(t *testing.T) { + dir := t.TempDir() + writeRepoFile(t, dir, "top.py", "1") + writeRepoFile(t, dir, "a/x.py", "1") + + assert.Equal(t, []string{"a/x.py", "top.py"}, selectedNames(t, dir, []string{"*.py"}, false)) + withoutGit(t) + assert.Equal(t, []string{"top.py"}, selectedNames(t, dir, []string{"*.py"}, false)) +} + func TestSnapshotFiles_RejectsIncludePathsMatchingNothing(t *testing.T) { repo := t.TempDir() writeRepoFile(t, repo, "a.txt", "1") diff --git a/cmd/air/snapshot_resolve.go b/cmd/air/snapshot_resolve.go index cec431796a6..4b7443f81e4 100644 --- a/cmd/air/snapshot_resolve.go +++ b/cmd/air/snapshot_resolve.go @@ -38,6 +38,20 @@ type snapshotPlan struct { subtreePrefix string } +// gitRequiredError is the error for packaging a code_source without a git binary, or +// nil when the git-free fallback may package it. The fallback applies only to a +// directory outside every repository with no git.* pin: packaging a repository's files +// there would ignore its rules, so it fails early instead. +func gitRequiredError(root string, ref *gitRef) error { + if ref != nil { + return fmt.Errorf("code_source.git is set, so packaging %s needs git on PATH; install git and retry", root) + } + if insideGitRepository(root) { + return fmt.Errorf("code_source at %s is in a git repository, so packaging it needs git on PATH; install git and retry", root) + } + return nil +} + // resolveSnapshotPlan decides how to package the snapshot (local-only): // - git.commit → pin the SHA (must exist locally) → git_archive. // - git.branch → the branch's local HEAD SHA → git_archive. @@ -49,7 +63,11 @@ type snapshotPlan struct { // git.branch is an error: the committed HEAD wouldn't include the uncommitted changes. func resolveSnapshotPlan(ctx context.Context, git gitRepo, ref *gitRef, includePaths []string) (snapshotPlan, error) { plan := snapshotPlan{includePaths: includePaths} - plan.isGitRepo = git.isRepository(ctx) + if gitInstalled() { + plan.isGitRepo = git.isRepository(ctx) + } else if err := gitRequiredError(git.path, ref); err != nil { + return snapshotPlan{}, err + } // Plain-tar snapshots include working-tree changes, and commit-pinned snapshots // intentionally exclude them, so neither needs a git status unless provenance diff --git a/cmd/air/snapshot_resolve_test.go b/cmd/air/snapshot_resolve_test.go index e903a2232de..275f03f9d48 100644 --- a/cmd/air/snapshot_resolve_test.go +++ b/cmd/air/snapshot_resolve_test.go @@ -1,6 +1,7 @@ package aircmd import ( + "os" "path/filepath" "testing" @@ -96,6 +97,52 @@ func TestResolveSnapshotPlan_NonGitDir(t *testing.T) { assert.Contains(t, err.Error(), "not a git repository") } +func TestResolveSnapshotPlan_WithoutGitNonRepoIsPlainTar(t *testing.T) { + dir := t.TempDir() + writeRepoFile(t, dir, "main.py", "1") + withoutGit(t) + + plan, err := resolveSnapshotPlan(t.Context(), newGitRepo(dir), nil, nil) + require.NoError(t, err) + assert.Equal(t, modePlainTar, plan.mode) + assert.False(t, plan.isGitRepo) +} + +// An empty .git directory is not a repository for git, so it must not trip the early error. +func TestResolveSnapshotPlan_WithoutGitEmptyDotGitIsNotRepo(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dir, ".git"), 0o755)) + withoutGit(t) + + plan, err := resolveSnapshotPlan(t.Context(), newGitRepo(dir), nil, nil) + require.NoError(t, err) + assert.Equal(t, modePlainTar, plan.mode) +} + +func TestResolveSnapshotPlan_WithoutGitInsideRepoFailsEarly(t *testing.T) { + repo := newTestRepo(t) + sub := filepath.Join(repo, "sub") + require.NoError(t, os.MkdirAll(sub, 0o755)) + withoutGit(t) + + _, err := resolveSnapshotPlan(t.Context(), newGitRepo(sub), nil, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "code_source at "+sub+" is in a git repository") + assert.Contains(t, err.Error(), "needs git on PATH") + assert.NotContains(t, err.Error(), "exec:") +} + +func TestResolveSnapshotPlan_WithoutGitRefFailsEarly(t *testing.T) { + dir := t.TempDir() + withoutGit(t) + + _, err := resolveSnapshotPlan(t.Context(), newGitRepo(dir), &gitRef{Branch: new("main")}, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "code_source.git is set") + assert.Contains(t, err.Error(), "needs git on PATH") + assert.NotContains(t, err.Error(), "exec:") +} + func TestResolveSnapshotPlan_IncludePaths(t *testing.T) { ctx := t.Context() repo := newTestRepo(t)