diff --git a/.nextchanges/air/non-git-snapshot-notice.md b/.nextchanges/air/non-git-snapshot-notice.md
new file mode 100644
index 00000000000..7047ab49222
--- /dev/null
+++ b/.nextchanges/air/non-git-snapshot-notice.md
@@ -0,0 +1 @@
+* `air run` from a directory that is not a git repository now says so before packaging, with the number of files and their uncompressed size (or that it reuses the cached snapshot), suggests `code_source.include_paths` or a `.gitignore` when the files total 64 MiB or more, and warns about listed paths it leaves out, such as nested git repositories. `--dry-run` no longer counts those paths. The notice is not printed with `-o json`. ([#7618](https://github.com/databricks/cli/pull/7618))
diff --git a/acceptance/air/run-submit-non-git/databricks.yml b/acceptance/air/run-submit-non-git/databricks.yml
new file mode 100644
index 00000000000..4a882a1d0ed
--- /dev/null
+++ b/acceptance/air/run-submit-non-git/databricks.yml
@@ -0,0 +1,2 @@
+bundle:
+ name: air-run-submit-non-git
diff --git a/acceptance/air/run-submit-non-git/out.test.toml b/acceptance/air/run-submit-non-git/out.test.toml
new file mode 100644
index 00000000000..18b1a88417e
--- /dev/null
+++ b/acceptance/air/run-submit-non-git/out.test.toml
@@ -0,0 +1 @@
+Cloud = false
diff --git a/acceptance/air/run-submit-non-git/output.txt b/acceptance/air/run-submit-non-git/output.txt
new file mode 100644
index 00000000000..c41b74e1844
--- /dev/null
+++ b/acceptance/air/run-submit-non-git/output.txt
@@ -0,0 +1,38 @@
+
+=== submit from a non-git directory
+
+>>> [CLI] air run -f run.yaml
+Submitting experiment: submit-non-git-smoke
+No git repository at [TEST_TMP_DIR]; packaging 5 file(s) (64.3 KiB uncompressed) from the working tree.
+Uploading [SNAPSHOT_TARBALL]...
+Submitted workload with Job Run ID: 555
+View job run at: [DATABRICKS_URL]/jobs/runs/555?o=[WORKSPACE_ID]
+
+Tip: use --watch when submitting a run to stream logs to your terminal.
+Stream logs after submission using:
+ databricks air logs 555
+
+=== unchanged resubmit reuses the cached snapshot
+
+>>> [CLI] air run -f run.yaml
+Submitting experiment: submit-non-git-smoke
+No git repository at [TEST_TMP_DIR]; reusing the cached snapshot (5 file(s), 64.3 KiB uncompressed).
+Submitted workload with Job Run ID: 555
+View job run at: [DATABRICKS_URL]/jobs/runs/555?o=[WORKSPACE_ID]
+
+Tip: use --watch when submitting a run to stream logs to your terminal.
+Stream logs after submission using:
+ databricks air logs 555
+
+=== json mode: no notice, like the Submitting line
+
+>>> [CLI] -o json air run -f run.yaml
+{
+ "v": 1,
+ "ts": "[TIMESTAMP]",
+ "data": {
+ "status": "PENDING",
+ "run_id": "555",
+ "dashboard_url": "[DATABRICKS_URL]/jobs/runs/555?o=[WORKSPACE_ID]"
+ }
+}
diff --git a/acceptance/air/run-submit-non-git/run.yaml b/acceptance/air/run-submit-non-git/run.yaml
new file mode 100644
index 00000000000..ff40a6f163b
--- /dev/null
+++ b/acceptance/air/run-submit-non-git/run.yaml
@@ -0,0 +1,9 @@
+experiment_name: submit-non-git-smoke
+command: python train.py
+compute:
+ accelerator_type: GPU_1xH100
+ num_accelerators: 1
+code_source:
+ type: snapshot
+ snapshot:
+ root_path: .
diff --git a/acceptance/air/run-submit-non-git/script b/acceptance/air/run-submit-non-git/script
new file mode 100644
index 00000000000..f636c255635
--- /dev/null
+++ b/acceptance/air/run-submit-non-git/script
@@ -0,0 +1,15 @@
+# No git-repo-init here: the directory stays non-git, so the snapshot must fall
+# back to plain_tar over everything in the working tree, including run.log and
+# the binary file, and the CLI announces that before packaging.
+echo "print('hello')" > main.py
+echo "some training log output" > run.log
+head -c 65536 /dev/zero > weights.bin
+
+title "submit from a non-git directory"
+trace $CLI air run -f run.yaml
+
+title "unchanged resubmit reuses the cached snapshot"
+trace $CLI air run -f run.yaml
+
+title "json mode: no notice, like the Submitting line"
+trace $CLI -o json air run -f run.yaml
diff --git a/acceptance/air/run-submit-non-git/test.toml b/acceptance/air/run-submit-non-git/test.toml
new file mode 100644
index 00000000000..2a4fbb30d8c
--- /dev/null
+++ b/acceptance/air/run-submit-non-git/test.toml
@@ -0,0 +1,37 @@
+# A real (non-dry-run) submit from a directory with no git repository: the
+# snapshot falls back to plain_tar over the whole working tree, and the CLI
+# says so (file count + uncompressed size) before packaging, including on the
+# second, cache-hit submit. -o json skips the notice.
+RecordRequests = true
+
+# main.py/run.log/weights.bin are generated by the script (the point of the
+# test is that a non-git code_source packages them); not committed inputs.
+Ignore = ["main.py", "run.log", "weights.bin"]
+
+# The SDK probes host reachability with a HEAD request; stub it for determinism.
+[[Server]]
+Pattern = "HEAD /"
+Response.Body = ''
+
+[[Server]]
+Pattern = "POST /api/2.0/ai-training/config:validate"
+Response.Body = '{}'
+
+[[Server]]
+Pattern = "POST /api/2.2/jobs/runs/submit"
+Response.Body = '''
+{"run_id": 555}
+'''
+
+# The snapshot tarball is named
_.tar.gz; the temp-dir basename
+# and the working-tree fingerprint vary per run, so collapse the name. The
+# notice's count and size are NOT masked: the committed inputs make them
+# deterministic, so the golden asserts them exactly.
+[[Repls]]
+Old = '[a-zA-Z0-9._-]+_[0-9a-f]{16}\.tar\.gz'
+New = '[SNAPSHOT_TARBALL]'
+
+# The per-run launch directory ends in _<16 hex>; the random suffix varies.
+[[Repls]]
+Old = 'submit-non-git-smoke_[0-9a-f]{16}'
+New = 'submit-non-git-smoke_[RUN_ID]'
diff --git a/cmd/air/runsubmit.go b/cmd/air/runsubmit.go
index c06165dd2b4..84d32a8c905 100644
--- a/cmd/air/runsubmit.go
+++ b/cmd/air/runsubmit.go
@@ -333,7 +333,7 @@ func stageRunArtifacts(ctx context.Context, launchWriter fileWriter, items []upl
// payload, and submit it. It
// returns the new run_id and its dashboard URL, which carries the ?o= workspace
// hint only when the workspace ID could be resolved. showProgress enables the stderr
-// staging spinner (text mode only).
+// staging spinner and the non-git snapshot notice (text mode only).
func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *runConfig, configPath, idempotencyKey string, showProgress bool) (runID int64, runURL string, err error) {
start := time.Now()
var snap snapshotResult
@@ -364,11 +364,20 @@ func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *run
var stageSnapshot func(context.Context) (snapshotResult, error)
if cfg.CodeSource != nil && cfg.CodeSource.Snapshot != nil {
// Default snapshot tarballs land in the user's shared repo_snapshots dir;
- // uploadSnapshot replaces this when remote_volume is configured.
+ // prepareSnapshot replaces this when remote_volume is configured.
snapshotArtifactPath := path.Join(base, ".air", "repo_snapshots")
+ // Resolve the snapshot before staging so its notice prints before the
+ // staging spinner starts drawing on stderr.
+ var prepared preparedSnapshot
+ prepared, err = prepareSnapshot(ctx, w, cfg.CodeSource.Snapshot, configPath, snapshotArtifactPath)
+ if err != nil {
+ snap = prepared.result
+ return 0, "", err
+ }
+ announceSnapshot(ctx, prepared, showProgress)
stageSnapshot = func(ctx context.Context) (snapshotResult, error) {
// Sidecars land in the run's launch dir (funcDir) via fc, next to command.sh.
- return uploadSnapshot(ctx, w, cfg.CodeSource.Snapshot, configPath, snapshotArtifactPath, fc, funcDir)
+ return prepared.upload(ctx, fc, funcDir)
}
}
diff --git a/cmd/air/snapshot_git.go b/cmd/air/snapshot_git.go
index 53d838ef9c0..104e2372f2d 100644
--- a/cmd/air/snapshot_git.go
+++ b/cmd/air/snapshot_git.go
@@ -10,6 +10,8 @@ import (
"os/exec"
"strings"
"time"
+
+ "github.com/databricks/cli/libs/log"
)
// Local, no-network git introspection and the git-state provenance sidecar, ported
@@ -68,6 +70,7 @@ func (g gitRepo) runTo(ctx context.Context, w io.Writer, args ...string) error {
func (g gitRepo) isRepository(ctx context.Context) bool {
out, err := g.run(ctx, "rev-parse", "--is-inside-work-tree")
if err != nil {
+ log.Debugf(ctx, "git repository probe for %s failed: %v", g.path, err)
return false
}
return strings.TrimSpace(out) == "true"
diff --git a/cmd/air/snapshot_notice_test.go b/cmd/air/snapshot_notice_test.go
new file mode 100644
index 00000000000..64a9af59990
--- /dev/null
+++ b/cmd/air/snapshot_notice_test.go
@@ -0,0 +1,261 @@
+package aircmd
+
+import (
+ "archive/tar"
+ "bytes"
+ "compress/gzip"
+ "errors"
+ "io"
+ "maps"
+ "os"
+ "path/filepath"
+ "slices"
+ "strings"
+ "sync"
+ "testing"
+
+ "github.com/databricks/cli/libs/cmdio"
+ "github.com/databricks/cli/libs/flags"
+ "github.com/databricks/cli/libs/testserver"
+ "github.com/databricks/databricks-sdk-go"
+ "github.com/databricks/databricks-sdk-go/service/jobs"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestPlainTarNoticeLines(t *testing.T) {
+ assert.Equal(t,
+ []string{"No git repository at /tmp/air-quickstart; packaging 3 file(s) (64.0 KiB uncompressed) from the working tree."},
+ plainTarNoticeLines("/tmp/air-quickstart", 3, 65576, false))
+ assert.Equal(t,
+ []string{"No git repository at /tmp/air-quickstart; reusing the cached snapshot (1 file(s), 15 B uncompressed)."},
+ plainTarNoticeLines("/tmp/air-quickstart", 1, 15, true))
+}
+
+func TestPlainTarNoticeLargeSourceTip(t *testing.T) {
+ const tip = "Tip: limit the snapshot with code_source.include_paths or a .gitignore in the directory."
+ assert.Len(t, plainTarNoticeLines("/x", 1, snapshotLargeSourceBytes-1, false), 1)
+
+ lines := plainTarNoticeLines("/x", 2, snapshotLargeSourceBytes, false)
+ assert.Equal(t, []string{"No git repository at /x; packaging 2 file(s) (64.0 MiB uncompressed) from the working tree.", tip}, lines)
+
+ // A cache hit packages nothing, so there is nothing to narrow.
+ assert.Len(t, plainTarNoticeLines("/x", 2, snapshotLargeSourceBytes, true), 1)
+}
+
+func TestShippedSnapshotFilesMatchesTarpack(t *testing.T) {
+ files := []snapshotFile{
+ {rel: "main.py", size: 15, mode: 0o644},
+ {rel: "link.py", size: 7, mode: uint32(os.ModeSymlink | 0o777), linkTarget: "main.py"},
+ {rel: "lib/", size: 4096, mode: uint32(os.ModeDir | 0o755)},
+ }
+ count, sourceBytes, skipped := shippedSnapshotFiles(files)
+ assert.Equal(t, 2, count)
+ // A symlink is stored as a link entry with no content.
+ assert.Equal(t, int64(15), sourceBytes)
+ assert.Equal(t, []string{"lib/"}, skipped)
+}
+
+func TestSkippedSnapshotWarningCapsNames(t *testing.T) {
+ assert.Equal(t,
+ "Warning: the snapshot leaves out 1 listed path(s) that are not regular files or symlinks, such as nested git repositories: lib/.",
+ skippedSnapshotWarning([]string{"lib/"}))
+ assert.Equal(t,
+ "Warning: the snapshot leaves out 7 listed path(s) that are not regular files or symlinks, such as nested git repositories: a/, b/, c/, d/, e/ (and 2 more).",
+ skippedSnapshotWarning([]string{"a/", "b/", "c/", "d/", "e/", "f/", "g/"}))
+}
+
+// syncBuffer is a goroutine-safe stderr: the spinner's renderer and the submit
+// path write to it concurrently.
+type syncBuffer struct {
+ mu sync.Mutex
+ buf bytes.Buffer
+}
+
+func (b *syncBuffer) Write(p []byte) (int, error) {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ return b.buf.Write(p)
+}
+
+func (b *syncBuffer) String() string {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ return b.buf.String()
+}
+
+// nonGitSubmitServer stubs a submit and keeps the uploaded snapshot tarballs by
+// path (the filer may retry a write after creating parent directories), so a test
+// can compare the notice against what actually shipped.
+func nonGitSubmitServer(t *testing.T) (*databricks.WorkspaceClient, func() [][]byte) {
+ t.Helper()
+ server := testserver.New(t)
+ t.Cleanup(server.Close)
+
+ var mu sync.Mutex
+ tarballs := map[string][]byte{}
+ server.Handle("POST", "/api/2.0/workspace-files/import-file/{path...}", func(req testserver.Request) any {
+ p := req.Vars["path"]
+ if strings.Contains(p, "/.air/repo_snapshots/") {
+ mu.Lock()
+ tarballs[p] = bytes.Clone(req.Body)
+ mu.Unlock()
+ }
+ return req.Workspace.WorkspaceFilesImportFile(p, req.Body, req.URL.Query().Get("overwrite") == "true")
+ })
+ server.Handle("POST", "/api/2.2/jobs/runs/submit", func(req testserver.Request) any {
+ return jobs.SubmitRunResponse{RunId: 555}
+ })
+ stubValidateConfig(server)
+ testserver.AddDefaultHandlers(server)
+
+ w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "token"})
+ require.NoError(t, err)
+ return w, func() [][]byte {
+ mu.Lock()
+ defer mu.Unlock()
+ return slices.Collect(maps.Values(tarballs))
+ }
+}
+
+func nonGitSnapshotConfig(t *testing.T, root string) (*runConfig, string) {
+ t.Helper()
+ cfgPath := writeConfigFile(t, "run.yaml", minimalConfig+`
+code_source:
+ type: snapshot
+ snapshot:
+ root_path: `+root+`
+`)
+ cfg, err := loadRunConfig(cfgPath)
+ require.NoError(t, err)
+ return cfg, cfgPath
+}
+
+func tarEntryNames(t *testing.T, tarball []byte) []string {
+ t.Helper()
+ gz, err := gzip.NewReader(bytes.NewReader(tarball))
+ require.NoError(t, err)
+ tr := tar.NewReader(gz)
+ var names []string
+ for {
+ hdr, err := tr.Next()
+ if errors.Is(err, io.EOF) {
+ return names
+ }
+ require.NoError(t, err)
+ names = append(names, hdr.Name)
+ }
+}
+
+// A nested git repository inside a non-git root is listed by ls-files as one
+// directory entry, which the tarball skips together with its files. The notice
+// must count what ships, and a warning must name the skipped directory.
+func TestSubmitWorkloadNonGitNestedRepo(t *testing.T) {
+ w, tarballs := nonGitSubmitServer(t)
+
+ root := filepath.Join(t.TempDir(), "proj")
+ writeRepoFile(t, root, "main.py", "print('hello')\n")
+ writeRepoFile(t, root, "run.log", "log\n")
+ writeRepoFile(t, root, "lib/x.py", strings.Repeat("x", 100))
+ runGit(t, filepath.Join(root, "lib"), "init", "-q")
+ cfg, cfgPath := nonGitSnapshotConfig(t, root)
+
+ var stderr syncBuffer
+ ctx := cmdio.InContext(t.Context(), cmdio.NewIO(t.Context(), flags.OutputText, strings.NewReader(""), io.Discard, &stderr, "", ""))
+ _, _, err := submitWorkload(ctx, w, cfg, cfgPath, "idem", true)
+ require.NoError(t, err)
+
+ uploaded := tarballs()
+ require.Len(t, uploaded, 1)
+ assert.ElementsMatch(t, []string{"proj/main.py", "proj/run.log"}, tarEntryNames(t, uploaded[0]))
+
+ out := stderr.String()
+ assert.Contains(t, out, "Warning: the snapshot leaves out 1 listed path(s) that are not regular files or symlinks, such as nested git repositories: lib/.\n")
+ assert.Contains(t, out, "No git repository at "+filepath.ToSlash(root)+"; packaging 2 file(s) (19 B uncompressed) from the working tree.\n")
+
+ // --dry-run reports the same count.
+ count, err := countSnapshotFiles(t.Context(), cfg.CodeSource.Snapshot, cfgPath)
+ require.NoError(t, err)
+ assert.Equal(t, 2, count)
+}
+
+// The notice prints before the staging spinner starts, so the spinner never
+// draws over it. cmdio.NewTestIO forces an interactive stderr, so the spinner
+// really renders: its first output (bubbletea's cursor-hiding escape sequence)
+// must come after the notice and the warning.
+func TestSubmitWorkloadNonGitNoticeBeforeSpinner(t *testing.T) {
+ w, _ := nonGitSubmitServer(t)
+
+ root := filepath.Join(t.TempDir(), "proj")
+ writeRepoFile(t, root, "main.py", "print('hello')\n")
+ writeRepoFile(t, root, "lib/x.py", "x")
+ runGit(t, filepath.Join(root, "lib"), "init", "-q")
+ cfg, cfgPath := nonGitSnapshotConfig(t, root)
+
+ var stderr syncBuffer
+ ctx := cmdio.InContext(t.Context(), cmdio.NewTestIO(strings.NewReader(""), io.Discard, &stderr))
+ _, _, err := submitWorkload(ctx, w, cfg, cfgPath, "idem", true)
+ require.NoError(t, err)
+
+ out := stderr.String()
+ warning := strings.Index(out, "Warning: the snapshot leaves out")
+ notice := strings.Index(out, "No git repository at ")
+ spinner := strings.Index(out, "\x1b[")
+ uploading := strings.Index(out, "Uploading ")
+ require.NotEqual(t, -1, warning, out)
+ require.NotEqual(t, -1, notice, out)
+ require.NotEqual(t, -1, spinner, "the spinner rendered nothing: %q", out)
+ require.NotEqual(t, -1, uploading, out)
+ assert.Less(t, warning, notice)
+ assert.Less(t, notice, spinner, "notice must precede the spinner: %q", out)
+ assert.Less(t, notice, uploading)
+}
+
+// An unchanged resubmit reuses the uploaded tarball and says so; under -o json
+// (showProgress=false) the notice is skipped like "Submitting experiment:", but
+// the skipped-path warning still prints.
+func TestSubmitWorkloadNonGitReuseAndJSON(t *testing.T) {
+ w, tarballs := nonGitSubmitServer(t)
+
+ root := filepath.Join(t.TempDir(), "proj")
+ writeRepoFile(t, root, "train.py", "print()")
+ cfg, cfgPath := nonGitSnapshotConfig(t, root)
+
+ submit := func(showProgress bool) string {
+ var stderr syncBuffer
+ ctx := cmdio.InContext(t.Context(), cmdio.NewIO(t.Context(), flags.OutputText, strings.NewReader(""), io.Discard, &stderr, "", ""))
+ _, _, err := submitWorkload(ctx, w, cfg, cfgPath, "idem", showProgress)
+ require.NoError(t, err)
+ return stderr.String()
+ }
+
+ assert.Contains(t, submit(true), "; packaging 1 file(s) (7 B uncompressed) from the working tree.")
+ reuse := submit(true)
+ assert.Contains(t, reuse, "No git repository at "+filepath.ToSlash(root)+"; reusing the cached snapshot (1 file(s), 7 B uncompressed).\n")
+ assert.NotContains(t, reuse, "packaging")
+ assert.Len(t, tarballs(), 1, "the resubmit must not upload again")
+
+ assert.NotContains(t, submit(false), "No git repository")
+
+ require.NoError(t, os.MkdirAll(filepath.Join(root, "lib"), 0o755))
+ runGit(t, filepath.Join(root, "lib"), "init", "-q")
+ writeRepoFile(t, root, "lib/x.py", "x")
+ jsonOut := submit(false)
+ assert.Contains(t, jsonOut, "Warning: the snapshot leaves out 1 listed path(s)")
+ assert.NotContains(t, jsonOut, "No git repository")
+}
+
+// A git repository gets no notice: git scopes its snapshot.
+func TestSubmitWorkloadGitRepoNoNotice(t *testing.T) {
+ w, _ := nonGitSubmitServer(t)
+ repo := newTestRepo(t)
+ writeRepoFile(t, repo, "train.py", "print()")
+ commitAll(t, repo, "init")
+ cfg, cfgPath := nonGitSnapshotConfig(t, repo)
+
+ var stderr syncBuffer
+ ctx := cmdio.InContext(t.Context(), cmdio.NewIO(t.Context(), flags.OutputText, strings.NewReader(""), io.Discard, &stderr, "", ""))
+ _, _, err := submitWorkload(ctx, w, cfg, cfgPath, "idem", true)
+ require.NoError(t, err)
+ assert.NotContains(t, stderr.String(), "No git repository")
+}
diff --git a/cmd/air/snapshot_upload.go b/cmd/air/snapshot_upload.go
index 209139c4055..04ab179c8da 100644
--- a/cmd/air/snapshot_upload.go
+++ b/cmd/air/snapshot_upload.go
@@ -15,6 +15,7 @@ import (
"github.com/databricks/cli/libs/cmdio"
"github.com/databricks/cli/libs/filer"
"github.com/databricks/cli/libs/log"
+ "github.com/databricks/cli/libs/tarpack"
"github.com/databricks/databricks-sdk-go"
)
@@ -24,14 +25,115 @@ import (
// latency to AIR submissions.
const uploadProvenanceSidecars = false
+// snapshotLargeSourceBytes is the uncompressed size of the shipped files at or
+// above which the non-git snapshot notice suggests narrowing the selection. The
+// value is an arbitrary default, not derived from measurements; tune it freely.
+const snapshotLargeSourceBytes = 64 << 20
+
+// snapshotSkippedPathsShown caps how many skipped paths the warning names.
+const snapshotSkippedPathsShown = 5
+
+// humanBytes formats a byte count using binary (1024-based) units. It is a copy
+// of humanBytes in cmd/fs/upload_progress.go, which is unexported there.
+func humanBytes(n int64) string {
+ const unit = 1024
+ if n < unit {
+ return fmt.Sprintf("%d B", n)
+ }
+ units := []string{"KiB", "MiB", "GiB", "TiB", "PiB"}
+ div, exp := int64(unit), 0
+ for n/div >= unit && exp < len(units)-1 {
+ div *= unit
+ exp++
+ }
+ return fmt.Sprintf("%.1f %s", float64(n)/float64(div), units[exp])
+}
+
+// shippedSnapshotFiles applies the plain tarball's entry rule (tarpack.Archives)
+// to a listing: it returns how many entries the tarball will hold, their
+// uncompressed content bytes (symlinks carry none), and the listed paths it will
+// skip. ls-files lists a nested git repository as one directory entry, so that
+// repository's files are skipped with it.
+func shippedSnapshotFiles(files []snapshotFile) (count int, sourceBytes int64, skipped []string) {
+ for _, f := range files {
+ mode := fs.FileMode(f.mode)
+ if !tarpack.Archives(mode) {
+ skipped = append(skipped, filepath.ToSlash(f.rel))
+ continue
+ }
+ count++
+ if mode.IsRegular() {
+ sourceBytes += f.size
+ }
+ }
+ return count, sourceBytes, skipped
+}
+
+// plainTarNoticeLines renders the non-git snapshot disclosure. The size is the
+// uncompressed shipped bytes, which include_paths and .gitignore control; the
+// compressed tarball size stays telemetry-only. reuse marks a cache hit, where
+// nothing is packaged or uploaded, so the size tip is omitted.
+func plainTarNoticeLines(repoPath string, count int, sourceBytes int64, reuse bool) []string {
+ base := filepath.ToSlash(repoPath)
+ total := humanBytes(sourceBytes)
+ if reuse {
+ return []string{fmt.Sprintf("No git repository at %s; reusing the cached snapshot (%d file(s), %s uncompressed).", base, count, total)}
+ }
+ lines := []string{fmt.Sprintf("No git repository at %s; packaging %d file(s) (%s uncompressed) from the working tree.", base, count, total)}
+ if sourceBytes >= snapshotLargeSourceBytes {
+ lines = append(lines, "Tip: limit the snapshot with code_source.include_paths or a .gitignore in the directory.")
+ }
+ return lines
+}
+
+// skippedSnapshotWarning names up to snapshotSkippedPathsShown listed paths that
+// the tarball leaves out.
+func skippedSnapshotWarning(skipped []string) string {
+ shown := skipped
+ more := ""
+ if len(shown) > snapshotSkippedPathsShown {
+ shown = shown[:snapshotSkippedPathsShown]
+ more = fmt.Sprintf(" (and %d more)", len(skipped)-snapshotSkippedPathsShown)
+ }
+ return fmt.Sprintf("Warning: the snapshot leaves out %d listed path(s) that are not regular files or symlinks, such as nested git repositories: %s%s.", len(skipped), strings.Join(shown, ", "), more)
+}
+
+// preparedSnapshot is a code_source snapshot resolved up to, but not including,
+// packaging: the plan, the content-addressed tarball name, the plain_tar listing
+// and whether the store already holds the tarball. submitWorkload resolves it
+// before the staging spinner starts so the snapshot notice prints first.
+type preparedSnapshot struct {
+ repoPath string
+ plan snapshotPlan
+ store filer.Filer
+ tarName string
+ remote string
+ // files is the plain_tar listing; nil for git_archive.
+ files []snapshotFile
+ exists bool
+ // result carries the telemetry fields known before packaging, and is
+ // returned on a resolution failure after the plan is known.
+ result snapshotResult
+}
+
// uploadSnapshot packages the code_source into a tarball and uploads it to the
// artifact store's .internal directory, returning the remote path to attach to the
// ai_runtime_task. Uploading reuses libs/filer for the workspace/volume write without
// depending on the bundle package (see uploadSnapshotTarball / snapshotUploadFiler).
func uploadSnapshot(ctx context.Context, w *databricks.WorkspaceClient, snap *snapshotSourceConfig, configPath, snapshotArtifactPath string, sidecarStore filer.Filer, sidecarBase string) (snapshotResult, error) {
+ p, err := prepareSnapshot(ctx, w, snap, configPath, snapshotArtifactPath)
+ if err != nil {
+ return p.result, err
+ }
+ return p.upload(ctx, sidecarStore, sidecarBase)
+}
+
+// prepareSnapshot resolves the root path, packaging plan and tarball name, and
+// checks whether the tarball is already uploaded.
+func prepareSnapshot(ctx context.Context, w *databricks.WorkspaceClient, snap *snapshotSourceConfig, configPath, snapshotArtifactPath string) (preparedSnapshot, error) {
repoPath, err := resolveRootPath(ctx, snap.RootPath, filepath.Dir(configPath))
if err != nil {
- return snapshotResult{}, err
+ return preparedSnapshot{}, err
}
// Resolve how to package before touching the tarball: git_archive (pinned commit) vs
@@ -39,7 +141,7 @@ func uploadSnapshot(ctx context.Context, w *databricks.WorkspaceClient, snap *sn
// the already-uploaded tarball instead of re-packaging and re-uploading.
plan, err := resolveSnapshotPlan(ctx, newGitRepo(repoPath), snap.Git, snap.IncludePaths)
if err != nil {
- return snapshotResult{}, err
+ return preparedSnapshot{}, err
}
// remote_volume, when set, is a UC Volume path; snapshotUploadFiler routes
@@ -47,7 +149,33 @@ func uploadSnapshot(ctx context.Context, w *databricks.WorkspaceClient, snap *sn
if snap.RemoteVolume != nil {
snapshotArtifactPath = *snap.RemoteVolume
}
- result, err := uploadSnapshotTarball(ctx, w, repoPath, plan, snapshotArtifactPath)
+ return prepareSnapshotTarball(ctx, w, repoPath, plan, snapshotArtifactPath)
+}
+
+// announceSnapshot tells the user what a non-git snapshot ships. The skipped-path
+// warning prints in every output mode, since those contents are silently missing
+// from the run; the notice is progress output and prints only when showNotice is
+// set (text mode). Git snapshots are scoped by git and get neither.
+func announceSnapshot(ctx context.Context, p preparedSnapshot, showNotice bool) {
+ if p.plan.mode != modePlainTar || p.plan.isGitRepo {
+ return
+ }
+ count, sourceBytes, skipped := shippedSnapshotFiles(p.files)
+ if len(skipped) > 0 {
+ cmdio.LogString(ctx, skippedSnapshotWarning(skipped))
+ }
+ if !showNotice {
+ return
+ }
+ for _, line := range plainTarNoticeLines(p.repoPath, count, sourceBytes, p.exists) {
+ cmdio.LogProgress(ctx, line)
+ }
+}
+
+// upload packages and uploads the prepared snapshot (unless it is already in the
+// store) and returns its remote code_source_path.
+func (p preparedSnapshot) upload(ctx context.Context, sidecarStore filer.Filer, sidecarBase string) (snapshotResult, error) {
+ result, err := p.uploadTarball(ctx)
if err != nil {
return result, err
}
@@ -65,8 +193,8 @@ func uploadSnapshot(ctx context.Context, w *databricks.WorkspaceClient, snap *sn
// cache) or serve a prior run's stale provenance on a cache hit. They also live in
// the per-run launch dir,
// not the shared artifact dir, so they don't accumulate. Keep them out of the tar.
- if uploadProvenanceSidecars && plan.isGitRepo {
- result.GitStatePath, result.GitDiffPath = uploadSnapshotSidecars(ctx, sidecarStore, sidecarBase, newGitRepo(repoPath), plan)
+ if uploadProvenanceSidecars && p.plan.isGitRepo {
+ result.GitStatePath, result.GitDiffPath = uploadSnapshotSidecars(ctx, sidecarStore, sidecarBase, newGitRepo(p.repoPath), p.plan)
}
return result, nil
}
@@ -133,7 +261,11 @@ func countSnapshotFiles(ctx context.Context, snap *snapshotSourceConfig, configP
return git.countFilesAtCommit(ctx, plan.commitSHA, plan.subtreePrefix, plan.includePaths)
}
files, err := snapshotFiles(ctx, repoPath, plan.includePaths, plan.isGitRepo)
- return len(files), err
+ if err != nil {
+ return 0, err
+ }
+ count, _, _ := shippedSnapshotFiles(files)
+ return count, nil
}
// snapshotTarName resolves the content-addressed upload filename for the snapshot and,
@@ -172,32 +304,49 @@ const airArtifactInternalDir = ".internal"
// uploadSnapshotTarball uploads the packaged snapshot to /.internal and returns
// the remote code_source_path to attach to the ai_runtime_task. artifactPath is either
// the user's repo_snapshots directory or a configured UC Volume.
-//
-// The tarball name is content-addressed — by (commit, include_paths, root_path subtree)
-// for git_archive and by the working-tree metadata fingerprint for plain_tar — so if
-// the identical object already exists we skip packaging and upload entirely and reuse it.
func uploadSnapshotTarball(ctx context.Context, w *databricks.WorkspaceClient, repoPath string, plan snapshotPlan, artifactPath string) (snapshotResult, error) {
- result := snapshotResult{UsesGit: &plan.isGitRepo, PackagingMode: &plan.mode}
+ p, err := prepareSnapshotTarball(ctx, w, repoPath, plan, artifactPath)
+ if err != nil {
+ return p.result, err
+ }
+ return p.uploadTarball(ctx)
+}
+
+// prepareSnapshotTarball resolves the upload store and the content-addressed tarball
+// name — by (commit, include_paths, root_path subtree) for git_archive and by the
+// working-tree metadata fingerprint for plain_tar — and checks whether the identical
+// object already exists, in which case uploadTarball skips packaging and upload.
+func prepareSnapshotTarball(ctx context.Context, w *databricks.WorkspaceClient, repoPath string, plan snapshotPlan, artifactPath string) (preparedSnapshot, error) {
+ p := preparedSnapshot{
+ repoPath: repoPath,
+ plan: plan,
+ result: snapshotResult{UsesGit: &plan.isGitRepo, PackagingMode: &plan.mode},
+ }
f, uploadPath, err := snapshotUploadFiler(ctx, w, artifactPath)
if err != nil {
- return result, err
+ return p, err
}
+ p.store = f
- tarName, files, err := snapshotTarName(ctx, repoPath, plan)
+ p.tarName, p.files, err = snapshotTarName(ctx, repoPath, plan)
if err != nil {
- return result, err
+ return p, err
}
// code_source_path is content-addressed by tarName, so it is the same whether we
// upload the bytes now or reuse an object already in the store.
- remote := path.Join(uploadPath, tarName)
+ p.remote = path.Join(uploadPath, p.tarName)
- exists, err := snapshotExists(ctx, f, tarName)
- if err != nil {
- return result, err
- }
- if exists {
- log.Debugf(ctx, "snapshot upload skipped; reusing %s", remote)
- result.CodeSourcePath = remote
+ p.exists, err = snapshotExists(ctx, f, p.tarName)
+ return p, err
+}
+
+// uploadTarball packages and uploads the prepared tarball, or reuses the stored one.
+func (p preparedSnapshot) uploadTarball(ctx context.Context) (snapshotResult, error) {
+ result := p.result
+ tarName := p.tarName
+ if p.exists {
+ log.Debugf(ctx, "snapshot upload skipped; reusing %s", p.remote)
+ result.CodeSourcePath = p.remote
result.PackagingDurationMs = new(int64(0))
result.UploadDurationMs = new(int64(0))
return result, nil
@@ -211,7 +360,7 @@ func uploadSnapshotTarball(ctx context.Context, w *databricks.WorkspaceClient, r
tarball := filepath.Join(tmp, tarName)
packagingStart := time.Now()
- err = packageSnapshot(ctx, repoPath, plan, files, tarball)
+ err = packageSnapshot(ctx, p.repoPath, p.plan, p.files, tarball)
result.PackagingDurationMs = new(time.Since(packagingStart).Milliseconds())
if err != nil {
return result, err
@@ -231,12 +380,12 @@ func uploadSnapshotTarball(ctx context.Context, w *databricks.WorkspaceClient, r
}
cmdio.LogProgress(ctx, fmt.Sprintf("Uploading %s...", tarName))
uploadStart := time.Now()
- err = f.Write(ctx, tarName, file, filer.OverwriteIfExists, filer.CreateParentDirectories)
+ err = p.store.Write(ctx, tarName, file, filer.OverwriteIfExists, filer.CreateParentDirectories)
result.UploadDurationMs = new(time.Since(uploadStart).Milliseconds())
if err != nil {
return result, fmt.Errorf("failed to upload snapshot %s: %w", tarName, err)
}
- result.CodeSourcePath = remote
+ result.CodeSourcePath = p.remote
return result, nil
}
diff --git a/libs/tarpack/tarpack.go b/libs/tarpack/tarpack.go
index 963988c0858..2139b3b0791 100644
--- a/libs/tarpack/tarpack.go
+++ b/libs/tarpack/tarpack.go
@@ -37,6 +37,12 @@ func Write(w io.Writer, entries []Entry) error {
return tw.Close()
}
+// Archives reports whether [Write] stores an entry of this file mode: regular
+// files and symlinks are archived, every other type is skipped.
+func Archives(mode fs.FileMode) bool {
+ return mode.IsRegular() || mode&fs.ModeSymlink != 0
+}
+
func writeEntry(tw *tar.Writer, e Entry) error {
// Lstat, not Stat, so a symlink is archived as a link rather than followed.
info, err := os.Lstat(e.Path)
@@ -44,7 +50,7 @@ func writeEntry(tw *tar.Writer, e Entry) error {
return fmt.Errorf("stat %s: %w", e.Path, err)
}
mode := info.Mode()
- if !mode.IsRegular() && mode&fs.ModeSymlink == 0 {
+ if !Archives(mode) {
return nil
}