Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/air/air-code-source-without-scm.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* Package `code_source` directories outside a git repository without a `git` binary on PATH, and fail early with a clear error when the directory is inside a repository or `code_source.git` is set. ([#7625](https://github.com/databricks/cli/pull/7625))
18 changes: 18 additions & 0 deletions cmd/air/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"io"
"os"
"os/signal"
"path/filepath"
"strconv"
"strings"
"time"
Expand All @@ -21,6 +22,20 @@ import (
"github.com/spf13/cobra"
)

// checkCodeSourceGit fails before the config is validated or anything is packaged when
// git is missing and the code_source cannot be packaged without it. Runs for dry runs too.
func checkCodeSourceGit(ctx context.Context, cfg *runConfig, configPath string) error {
if cfg.CodeSource == nil || cfg.CodeSource.Snapshot == nil || gitInstalled() {
return nil
}
snap := cfg.CodeSource.Snapshot
root, err := resolveRootPath(ctx, snap.RootPath, filepath.Dir(configPath))
if err != nil {
return err
}
return gitRequiredError(root, snap.Git)
}

// dryRunValidationTimeout bounds only the config:validate request.
const dryRunValidationTimeout = 15 * time.Second

Expand Down Expand Up @@ -159,6 +174,9 @@ field, pass its path to -h as a separate argument:
if err != nil {
return err
}
if err := checkCodeSourceGit(ctx, cfg, file); err != nil {
return err
}

if dryRun {
// Local and credential-free, so it runs before workspace validation.
Expand Down
37 changes: 37 additions & 0 deletions cmd/air/run_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@ import (
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
"strings"
"sync/atomic"
"testing"
Expand Down Expand Up @@ -87,6 +90,40 @@ func TestRunSubmitTextOutput(t *testing.T) {
assert.Zero(t, counts.runGetOutput.Load(), "bare submission must not poll runs/get-output")
}

func TestCheckCodeSourceGit(t *testing.T) {
repo := newTestRepo(t)
sub := filepath.Join(repo, "sub")
require.NoError(t, os.MkdirAll(sub, 0o755))
plain := t.TempDir()
withCodeSource := func(root string, ref *gitRef) *runConfig {
return &runConfig{CodeSource: &codeSourceConfig{Snapshot: &snapshotSourceConfig{RootPath: root, Git: ref}}}
}
withoutGit(t)

err := checkCodeSourceGit(t.Context(), withCodeSource(".", nil), filepath.Join(sub, "workload.yaml"))
require.Error(t, err)
assert.Contains(t, err.Error(), "is in a git repository")
assert.Contains(t, err.Error(), "needs git on PATH")
assert.NotContains(t, err.Error(), "exec:")

err = checkCodeSourceGit(t.Context(), withCodeSource(".", &gitRef{Branch: new("main")}), filepath.Join(plain, "workload.yaml"))
require.Error(t, err)
assert.Contains(t, err.Error(), "code_source.git is set")

require.NoError(t, checkCodeSourceGit(t.Context(), withCodeSource(".", nil), filepath.Join(plain, "workload.yaml")))

// Without code_source the git binary is never looked up, and its absence is not an error.
looked := false
orig := gitLookPath
gitLookPath = func(string) (string, error) {
looked = true
return "", exec.ErrNotFound
}
t.Cleanup(func() { gitLookPath = orig })
require.NoError(t, checkCodeSourceGit(t.Context(), &runConfig{}, filepath.Join(plain, "workload.yaml")))
assert.False(t, looked)
}

func TestWarnExperimentalContainers(t *testing.T) {
var stdout, stderr bytes.Buffer
ctx := cmdio.InContext(t.Context(), cmdio.NewIO(t.Context(), flags.OutputText, nil, &stdout, &stderr, "", ""))
Expand Down
38 changes: 38 additions & 0 deletions cmd/air/snapshot_git.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,9 @@ import (
"errors"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
)
Expand Down Expand Up @@ -73,6 +75,42 @@ func (g gitRepo) isRepository(ctx context.Context) bool {
return strings.TrimSpace(out) == "true"
}

// hasGitMetadata reports whether dir holds a repository's `.git` entry: a gitfile, or a
// directory with HEAD. An empty `.git` directory is not a repository, as in git.
func hasGitMetadata(dir string) bool {
info, err := os.Stat(filepath.Join(dir, ".git"))
if err != nil {
return false
}
if info.Mode().IsRegular() {
return true
}
if !info.IsDir() {
return false
}
_, err = os.Stat(filepath.Join(dir, ".git", "HEAD"))
return err == nil
}

// insideGitRepository is the git-free stand-in for isRepository, used only when no git
// binary is available: repository metadata in dir or one of its ancestors marks a work tree.
func insideGitRepository(dir string) bool {
abs, err := filepath.Abs(dir)
if err != nil {
return false
}
for {
if hasGitMetadata(abs) {
return true
}
parent := filepath.Dir(abs)
if parent == abs {
return false
}
abs = parent
}
}

// repoRelativePrefix returns the path from the repository root to g.path. Git
// emits a trailing slash for subdirectories and an empty string at the root.
func (g gitRepo) repoRelativePrefix(ctx context.Context) (string, error) {
Expand Down
177 changes: 153 additions & 24 deletions cmd/air/snapshot_package.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,17 @@ import (
"errors"
"fmt"
"io"
"io/fs"
"os"
"os/exec"
"path"
"path/filepath"
"slices"
"strings"

libgit "github.com/databricks/cli/libs/git"
"github.com/databricks/cli/libs/tarpack"
"github.com/databricks/cli/libs/vfs"
"github.com/klauspost/pgzip"
)

Expand Down Expand Up @@ -135,35 +140,20 @@ type snapshotFile struct {
}

func snapshotFiles(ctx context.Context, repoPath string, includePaths []string, isGitRepo bool) ([]snapshotFile, error) {
args := []string{"-C", repoPath, "ls-files", "-z", "--cached", "--others", "--exclude-standard"}
if !isGitRepo {
gitDir, err := os.MkdirTemp("", "air-snapshot-git-")
if err != nil {
return nil, fmt.Errorf("failed to create temporary git metadata: %w", err)
}
defer os.RemoveAll(gitDir)
cmd := exec.CommandContext(ctx, "git", "--git-dir", gitDir, "--work-tree", repoPath, "init", "--quiet")
if output, err := cmd.CombinedOutput(); err != nil {
return nil, fmt.Errorf("failed to initialize temporary git metadata: %w: %s", err, strings.TrimSpace(string(output)))
}
args = []string{"--git-dir", gitDir, "--work-tree", repoPath, "ls-files", "-z", "--cached", "--others", "--exclude-standard"}
var names []string
var err error
if !isGitRepo && !gitInstalled() {
names, err = listFilesWithoutGit(ctx, repoPath, includePaths)
} else {
names, err = listFilesWithGit(ctx, repoPath, includePaths, isGitRepo)
}

if len(includePaths) > 0 {
args = append(args, "--")
args = append(args, includePaths...)
}
output, err := exec.CommandContext(ctx, "git", args...).Output()
if err != nil {
return nil, fmt.Errorf("failed to evaluate git ignore rules: %w", err)
return nil, err
}

var files []snapshotFile
for raw := range bytes.SplitSeq(output, []byte{0}) {
if len(raw) == 0 {
continue
}
name := filepath.ToSlash(string(raw))
for _, raw := range names {
name := filepath.ToSlash(raw)
base := filepath.Base(name)
if name == ".git" || strings.HasPrefix(name, ".git/") || strings.HasPrefix(base, "._") {
continue
Expand Down Expand Up @@ -196,6 +186,145 @@ func snapshotFiles(ctx context.Context, repoPath string, includePaths []string,
return files, nil
}

// listFilesWithGit returns the repo-relative paths `git ls-files` selects for repoPath.
// A directory outside any repository gets a temporary git dir so the same ignore
// rules apply without touching the directory.
func listFilesWithGit(ctx context.Context, repoPath string, includePaths []string, isGitRepo bool) ([]string, error) {
args := []string{"-C", repoPath, "ls-files", "-z", "--cached", "--others", "--exclude-standard"}
if !isGitRepo {
gitDir, err := os.MkdirTemp("", "air-snapshot-git-")
if err != nil {
return nil, fmt.Errorf("failed to create temporary git metadata: %w", err)
}
defer os.RemoveAll(gitDir)
cmd := exec.CommandContext(ctx, "git", "--git-dir", gitDir, "--work-tree", repoPath, "init", "--quiet")
if output, err := cmd.CombinedOutput(); err != nil {
return nil, fmt.Errorf("failed to initialize temporary git metadata: %w: %s", err, strings.TrimSpace(string(output)))
}
args = []string{"--git-dir", gitDir, "--work-tree", repoPath, "ls-files", "-z", "--cached", "--others", "--exclude-standard"}
}

if len(includePaths) > 0 {
args = append(args, "--")
args = append(args, includePaths...)
}
output, err := exec.CommandContext(ctx, "git", args...).Output()
if err != nil {
return nil, fmt.Errorf("failed to evaluate git ignore rules: %w", err)
}

var names []string
for raw := range bytes.SplitSeq(output, []byte{0}) {
if len(raw) > 0 {
names = append(names, string(raw))
}
}
return names, nil
}

// gitLookPath finds the git binary. Tests replace it to simulate a host without git.
var gitLookPath = exec.LookPath

// gitInstalled reports whether a git binary is on PATH.
func gitInstalled() bool {
_, err := gitLookPath("git")
return err == nil
}

// listFilesWithoutGit is the fallback for a directory outside any repository when git
// is not on PATH. It walks the tree applying libs/git's ignore rules (the root and
// nested .gitignore files, info/exclude and core.excludesFile from config), lists
// regular files and symlinks without following symlinks, and emits a nested repository
// as one "dir/" entry, as git does.
//
// Known differences from `git ls-files --others --exclude-standard`, which only matter
// on hosts without git (with git on PATH the git listing is used unchanged):
// - Rule precedence: libs/git returns the first matching rule from the root down, so
// a root "*.log" excludes nested/keep.log even when nested/.gitignore has "!keep.log".
// - include_paths use path.Match, where "*" does not cross "/"; git's pathspecs do.
func listFilesWithoutGit(ctx context.Context, repoPath string, includePaths []string) ([]string, error) {
repo, err := libgit.NewRepository(ctx, vfs.MustNew(repoPath))
if err != nil {
return nil, fmt.Errorf("failed to read .gitignore rules: %w", err)
}

var names []string
err = filepath.WalkDir(repoPath, func(p string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if p == repoPath {
return nil
}
rel, err := filepath.Rel(repoPath, p)
if err != nil {
return err
}
name := filepath.ToSlash(rel)
if d.Name() == ".git" {
if d.IsDir() {
return fs.SkipDir
}
return nil
}
if d.IsDir() {
ignored, err := repo.Ignore(name + "/")
if err != nil {
return err
}
if ignored {
return fs.SkipDir
}
if hasGitMetadata(p) {
names = append(names, name+"/")
return fs.SkipDir
}
return nil
}
ignored, err := repo.Ignore(name)
if err != nil {
return err
}
if !ignored {
names = append(names, name)
}
return nil
})
if err != nil {
return nil, fmt.Errorf("failed to list files: %w", err)
}

slices.Sort(names)
if len(includePaths) > 0 {
names = slices.DeleteFunc(names, func(name string) bool {
return !matchesIncludePath(name, includePaths)
})
}
return names, nil
}

// matchesIncludePath reports whether name, or a directory containing it, matches one of
// the include_paths.
func matchesIncludePath(name string, includePaths []string) bool {
for _, spec := range includePaths {
spec = path.Clean(filepath.ToSlash(strings.TrimSpace(spec)))
if spec == "." {
return true
}
for p := strings.TrimSuffix(name, "/"); p != ""; {
if ok, _ := path.Match(spec, p); ok {
return true
}
i := strings.LastIndex(p, "/")
if i < 0 {
break
}
p = p[:i]
}
}
return false
}

// emptySnapshotError explains why no file was selected for the snapshot. An empty
// tarball is accepted by the upload but fails on the node, after compute was
// provisioned, so it is rejected here with the most likely cause.
Expand Down
Loading
Loading