diff --git a/.nextchanges/air/non-git-snapshot-notice.md b/.nextchanges/air/non-git-snapshot-notice.md new file mode 100644 index 00000000000..7047ab49222 --- /dev/null +++ b/.nextchanges/air/non-git-snapshot-notice.md @@ -0,0 +1 @@ +* `air run` from a directory that is not a git repository now says so before packaging, with the number of files and their uncompressed size (or that it reuses the cached snapshot), suggests `code_source.include_paths` or a `.gitignore` when the files total 64 MiB or more, and warns about listed paths it leaves out, such as nested git repositories. `--dry-run` no longer counts those paths. The notice is not printed with `-o json`. ([#7618](https://github.com/databricks/cli/pull/7618)) diff --git a/acceptance/air/run-submit-non-git/databricks.yml b/acceptance/air/run-submit-non-git/databricks.yml new file mode 100644 index 00000000000..4a882a1d0ed --- /dev/null +++ b/acceptance/air/run-submit-non-git/databricks.yml @@ -0,0 +1,2 @@ +bundle: + name: air-run-submit-non-git diff --git a/acceptance/air/run-submit-non-git/out.test.toml b/acceptance/air/run-submit-non-git/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/air/run-submit-non-git/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/air/run-submit-non-git/output.txt b/acceptance/air/run-submit-non-git/output.txt new file mode 100644 index 00000000000..c41b74e1844 --- /dev/null +++ b/acceptance/air/run-submit-non-git/output.txt @@ -0,0 +1,38 @@ + +=== submit from a non-git directory + +>>> [CLI] air run -f run.yaml +Submitting experiment: submit-non-git-smoke +No git repository at [TEST_TMP_DIR]; packaging 5 file(s) (64.3 KiB uncompressed) from the working tree. +Uploading [SNAPSHOT_TARBALL]... +Submitted workload with Job Run ID: 555 +View job run at: [DATABRICKS_URL]/jobs/runs/555?o=[WORKSPACE_ID] + +Tip: use --watch when submitting a run to stream logs to your terminal. +Stream logs after submission using: + databricks air logs 555 + +=== unchanged resubmit reuses the cached snapshot + +>>> [CLI] air run -f run.yaml +Submitting experiment: submit-non-git-smoke +No git repository at [TEST_TMP_DIR]; reusing the cached snapshot (5 file(s), 64.3 KiB uncompressed). +Submitted workload with Job Run ID: 555 +View job run at: [DATABRICKS_URL]/jobs/runs/555?o=[WORKSPACE_ID] + +Tip: use --watch when submitting a run to stream logs to your terminal. +Stream logs after submission using: + databricks air logs 555 + +=== json mode: no notice, like the Submitting line + +>>> [CLI] -o json air run -f run.yaml +{ + "v": 1, + "ts": "[TIMESTAMP]", + "data": { + "status": "PENDING", + "run_id": "555", + "dashboard_url": "[DATABRICKS_URL]/jobs/runs/555?o=[WORKSPACE_ID]" + } +} diff --git a/acceptance/air/run-submit-non-git/run.yaml b/acceptance/air/run-submit-non-git/run.yaml new file mode 100644 index 00000000000..ff40a6f163b --- /dev/null +++ b/acceptance/air/run-submit-non-git/run.yaml @@ -0,0 +1,9 @@ +experiment_name: submit-non-git-smoke +command: python train.py +compute: + accelerator_type: GPU_1xH100 + num_accelerators: 1 +code_source: + type: snapshot + snapshot: + root_path: . diff --git a/acceptance/air/run-submit-non-git/script b/acceptance/air/run-submit-non-git/script new file mode 100644 index 00000000000..f636c255635 --- /dev/null +++ b/acceptance/air/run-submit-non-git/script @@ -0,0 +1,15 @@ +# No git-repo-init here: the directory stays non-git, so the snapshot must fall +# back to plain_tar over everything in the working tree, including run.log and +# the binary file, and the CLI announces that before packaging. +echo "print('hello')" > main.py +echo "some training log output" > run.log +head -c 65536 /dev/zero > weights.bin + +title "submit from a non-git directory" +trace $CLI air run -f run.yaml + +title "unchanged resubmit reuses the cached snapshot" +trace $CLI air run -f run.yaml + +title "json mode: no notice, like the Submitting line" +trace $CLI -o json air run -f run.yaml diff --git a/acceptance/air/run-submit-non-git/test.toml b/acceptance/air/run-submit-non-git/test.toml new file mode 100644 index 00000000000..2a4fbb30d8c --- /dev/null +++ b/acceptance/air/run-submit-non-git/test.toml @@ -0,0 +1,37 @@ +# A real (non-dry-run) submit from a directory with no git repository: the +# snapshot falls back to plain_tar over the whole working tree, and the CLI +# says so (file count + uncompressed size) before packaging, including on the +# second, cache-hit submit. -o json skips the notice. +RecordRequests = true + +# main.py/run.log/weights.bin are generated by the script (the point of the +# test is that a non-git code_source packages them); not committed inputs. +Ignore = ["main.py", "run.log", "weights.bin"] + +# The SDK probes host reachability with a HEAD request; stub it for determinism. +[[Server]] +Pattern = "HEAD /" +Response.Body = '' + +[[Server]] +Pattern = "POST /api/2.0/ai-training/config:validate" +Response.Body = '{}' + +[[Server]] +Pattern = "POST /api/2.2/jobs/runs/submit" +Response.Body = ''' +{"run_id": 555} +''' + +# The snapshot tarball is named _.tar.gz; the temp-dir basename +# and the working-tree fingerprint vary per run, so collapse the name. The +# notice's count and size are NOT masked: the committed inputs make them +# deterministic, so the golden asserts them exactly. +[[Repls]] +Old = '[a-zA-Z0-9._-]+_[0-9a-f]{16}\.tar\.gz' +New = '[SNAPSHOT_TARBALL]' + +# The per-run launch directory ends in _<16 hex>; the random suffix varies. +[[Repls]] +Old = 'submit-non-git-smoke_[0-9a-f]{16}' +New = 'submit-non-git-smoke_[RUN_ID]' diff --git a/cmd/air/runsubmit.go b/cmd/air/runsubmit.go index c06165dd2b4..84d32a8c905 100644 --- a/cmd/air/runsubmit.go +++ b/cmd/air/runsubmit.go @@ -333,7 +333,7 @@ func stageRunArtifacts(ctx context.Context, launchWriter fileWriter, items []upl // payload, and submit it. It // returns the new run_id and its dashboard URL, which carries the ?o= workspace // hint only when the workspace ID could be resolved. showProgress enables the stderr -// staging spinner (text mode only). +// staging spinner and the non-git snapshot notice (text mode only). func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *runConfig, configPath, idempotencyKey string, showProgress bool) (runID int64, runURL string, err error) { start := time.Now() var snap snapshotResult @@ -364,11 +364,20 @@ func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *run var stageSnapshot func(context.Context) (snapshotResult, error) if cfg.CodeSource != nil && cfg.CodeSource.Snapshot != nil { // Default snapshot tarballs land in the user's shared repo_snapshots dir; - // uploadSnapshot replaces this when remote_volume is configured. + // prepareSnapshot replaces this when remote_volume is configured. snapshotArtifactPath := path.Join(base, ".air", "repo_snapshots") + // Resolve the snapshot before staging so its notice prints before the + // staging spinner starts drawing on stderr. + var prepared preparedSnapshot + prepared, err = prepareSnapshot(ctx, w, cfg.CodeSource.Snapshot, configPath, snapshotArtifactPath) + if err != nil { + snap = prepared.result + return 0, "", err + } + announceSnapshot(ctx, prepared, showProgress) stageSnapshot = func(ctx context.Context) (snapshotResult, error) { // Sidecars land in the run's launch dir (funcDir) via fc, next to command.sh. - return uploadSnapshot(ctx, w, cfg.CodeSource.Snapshot, configPath, snapshotArtifactPath, fc, funcDir) + return prepared.upload(ctx, fc, funcDir) } } diff --git a/cmd/air/snapshot_git.go b/cmd/air/snapshot_git.go index 53d838ef9c0..104e2372f2d 100644 --- a/cmd/air/snapshot_git.go +++ b/cmd/air/snapshot_git.go @@ -10,6 +10,8 @@ import ( "os/exec" "strings" "time" + + "github.com/databricks/cli/libs/log" ) // Local, no-network git introspection and the git-state provenance sidecar, ported @@ -68,6 +70,7 @@ func (g gitRepo) runTo(ctx context.Context, w io.Writer, args ...string) error { func (g gitRepo) isRepository(ctx context.Context) bool { out, err := g.run(ctx, "rev-parse", "--is-inside-work-tree") if err != nil { + log.Debugf(ctx, "git repository probe for %s failed: %v", g.path, err) return false } return strings.TrimSpace(out) == "true" diff --git a/cmd/air/snapshot_notice_test.go b/cmd/air/snapshot_notice_test.go new file mode 100644 index 00000000000..64a9af59990 --- /dev/null +++ b/cmd/air/snapshot_notice_test.go @@ -0,0 +1,261 @@ +package aircmd + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "errors" + "io" + "maps" + "os" + "path/filepath" + "slices" + "strings" + "sync" + "testing" + + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/cli/libs/testserver" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPlainTarNoticeLines(t *testing.T) { + assert.Equal(t, + []string{"No git repository at /tmp/air-quickstart; packaging 3 file(s) (64.0 KiB uncompressed) from the working tree."}, + plainTarNoticeLines("/tmp/air-quickstart", 3, 65576, false)) + assert.Equal(t, + []string{"No git repository at /tmp/air-quickstart; reusing the cached snapshot (1 file(s), 15 B uncompressed)."}, + plainTarNoticeLines("/tmp/air-quickstart", 1, 15, true)) +} + +func TestPlainTarNoticeLargeSourceTip(t *testing.T) { + const tip = "Tip: limit the snapshot with code_source.include_paths or a .gitignore in the directory." + assert.Len(t, plainTarNoticeLines("/x", 1, snapshotLargeSourceBytes-1, false), 1) + + lines := plainTarNoticeLines("/x", 2, snapshotLargeSourceBytes, false) + assert.Equal(t, []string{"No git repository at /x; packaging 2 file(s) (64.0 MiB uncompressed) from the working tree.", tip}, lines) + + // A cache hit packages nothing, so there is nothing to narrow. + assert.Len(t, plainTarNoticeLines("/x", 2, snapshotLargeSourceBytes, true), 1) +} + +func TestShippedSnapshotFilesMatchesTarpack(t *testing.T) { + files := []snapshotFile{ + {rel: "main.py", size: 15, mode: 0o644}, + {rel: "link.py", size: 7, mode: uint32(os.ModeSymlink | 0o777), linkTarget: "main.py"}, + {rel: "lib/", size: 4096, mode: uint32(os.ModeDir | 0o755)}, + } + count, sourceBytes, skipped := shippedSnapshotFiles(files) + assert.Equal(t, 2, count) + // A symlink is stored as a link entry with no content. + assert.Equal(t, int64(15), sourceBytes) + assert.Equal(t, []string{"lib/"}, skipped) +} + +func TestSkippedSnapshotWarningCapsNames(t *testing.T) { + assert.Equal(t, + "Warning: the snapshot leaves out 1 listed path(s) that are not regular files or symlinks, such as nested git repositories: lib/.", + skippedSnapshotWarning([]string{"lib/"})) + assert.Equal(t, + "Warning: the snapshot leaves out 7 listed path(s) that are not regular files or symlinks, such as nested git repositories: a/, b/, c/, d/, e/ (and 2 more).", + skippedSnapshotWarning([]string{"a/", "b/", "c/", "d/", "e/", "f/", "g/"})) +} + +// syncBuffer is a goroutine-safe stderr: the spinner's renderer and the submit +// path write to it concurrently. +type syncBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (b *syncBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.Write(p) +} + +func (b *syncBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.String() +} + +// nonGitSubmitServer stubs a submit and keeps the uploaded snapshot tarballs by +// path (the filer may retry a write after creating parent directories), so a test +// can compare the notice against what actually shipped. +func nonGitSubmitServer(t *testing.T) (*databricks.WorkspaceClient, func() [][]byte) { + t.Helper() + server := testserver.New(t) + t.Cleanup(server.Close) + + var mu sync.Mutex + tarballs := map[string][]byte{} + server.Handle("POST", "/api/2.0/workspace-files/import-file/{path...}", func(req testserver.Request) any { + p := req.Vars["path"] + if strings.Contains(p, "/.air/repo_snapshots/") { + mu.Lock() + tarballs[p] = bytes.Clone(req.Body) + mu.Unlock() + } + return req.Workspace.WorkspaceFilesImportFile(p, req.Body, req.URL.Query().Get("overwrite") == "true") + }) + server.Handle("POST", "/api/2.2/jobs/runs/submit", func(req testserver.Request) any { + return jobs.SubmitRunResponse{RunId: 555} + }) + stubValidateConfig(server) + testserver.AddDefaultHandlers(server) + + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "token"}) + require.NoError(t, err) + return w, func() [][]byte { + mu.Lock() + defer mu.Unlock() + return slices.Collect(maps.Values(tarballs)) + } +} + +func nonGitSnapshotConfig(t *testing.T, root string) (*runConfig, string) { + t.Helper() + cfgPath := writeConfigFile(t, "run.yaml", minimalConfig+` +code_source: + type: snapshot + snapshot: + root_path: `+root+` +`) + cfg, err := loadRunConfig(cfgPath) + require.NoError(t, err) + return cfg, cfgPath +} + +func tarEntryNames(t *testing.T, tarball []byte) []string { + t.Helper() + gz, err := gzip.NewReader(bytes.NewReader(tarball)) + require.NoError(t, err) + tr := tar.NewReader(gz) + var names []string + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + return names + } + require.NoError(t, err) + names = append(names, hdr.Name) + } +} + +// A nested git repository inside a non-git root is listed by ls-files as one +// directory entry, which the tarball skips together with its files. The notice +// must count what ships, and a warning must name the skipped directory. +func TestSubmitWorkloadNonGitNestedRepo(t *testing.T) { + w, tarballs := nonGitSubmitServer(t) + + root := filepath.Join(t.TempDir(), "proj") + writeRepoFile(t, root, "main.py", "print('hello')\n") + writeRepoFile(t, root, "run.log", "log\n") + writeRepoFile(t, root, "lib/x.py", strings.Repeat("x", 100)) + runGit(t, filepath.Join(root, "lib"), "init", "-q") + cfg, cfgPath := nonGitSnapshotConfig(t, root) + + var stderr syncBuffer + ctx := cmdio.InContext(t.Context(), cmdio.NewIO(t.Context(), flags.OutputText, strings.NewReader(""), io.Discard, &stderr, "", "")) + _, _, err := submitWorkload(ctx, w, cfg, cfgPath, "idem", true) + require.NoError(t, err) + + uploaded := tarballs() + require.Len(t, uploaded, 1) + assert.ElementsMatch(t, []string{"proj/main.py", "proj/run.log"}, tarEntryNames(t, uploaded[0])) + + out := stderr.String() + assert.Contains(t, out, "Warning: the snapshot leaves out 1 listed path(s) that are not regular files or symlinks, such as nested git repositories: lib/.\n") + assert.Contains(t, out, "No git repository at "+filepath.ToSlash(root)+"; packaging 2 file(s) (19 B uncompressed) from the working tree.\n") + + // --dry-run reports the same count. + count, err := countSnapshotFiles(t.Context(), cfg.CodeSource.Snapshot, cfgPath) + require.NoError(t, err) + assert.Equal(t, 2, count) +} + +// The notice prints before the staging spinner starts, so the spinner never +// draws over it. cmdio.NewTestIO forces an interactive stderr, so the spinner +// really renders: its first output (bubbletea's cursor-hiding escape sequence) +// must come after the notice and the warning. +func TestSubmitWorkloadNonGitNoticeBeforeSpinner(t *testing.T) { + w, _ := nonGitSubmitServer(t) + + root := filepath.Join(t.TempDir(), "proj") + writeRepoFile(t, root, "main.py", "print('hello')\n") + writeRepoFile(t, root, "lib/x.py", "x") + runGit(t, filepath.Join(root, "lib"), "init", "-q") + cfg, cfgPath := nonGitSnapshotConfig(t, root) + + var stderr syncBuffer + ctx := cmdio.InContext(t.Context(), cmdio.NewTestIO(strings.NewReader(""), io.Discard, &stderr)) + _, _, err := submitWorkload(ctx, w, cfg, cfgPath, "idem", true) + require.NoError(t, err) + + out := stderr.String() + warning := strings.Index(out, "Warning: the snapshot leaves out") + notice := strings.Index(out, "No git repository at ") + spinner := strings.Index(out, "\x1b[") + uploading := strings.Index(out, "Uploading ") + require.NotEqual(t, -1, warning, out) + require.NotEqual(t, -1, notice, out) + require.NotEqual(t, -1, spinner, "the spinner rendered nothing: %q", out) + require.NotEqual(t, -1, uploading, out) + assert.Less(t, warning, notice) + assert.Less(t, notice, spinner, "notice must precede the spinner: %q", out) + assert.Less(t, notice, uploading) +} + +// An unchanged resubmit reuses the uploaded tarball and says so; under -o json +// (showProgress=false) the notice is skipped like "Submitting experiment:", but +// the skipped-path warning still prints. +func TestSubmitWorkloadNonGitReuseAndJSON(t *testing.T) { + w, tarballs := nonGitSubmitServer(t) + + root := filepath.Join(t.TempDir(), "proj") + writeRepoFile(t, root, "train.py", "print()") + cfg, cfgPath := nonGitSnapshotConfig(t, root) + + submit := func(showProgress bool) string { + var stderr syncBuffer + ctx := cmdio.InContext(t.Context(), cmdio.NewIO(t.Context(), flags.OutputText, strings.NewReader(""), io.Discard, &stderr, "", "")) + _, _, err := submitWorkload(ctx, w, cfg, cfgPath, "idem", showProgress) + require.NoError(t, err) + return stderr.String() + } + + assert.Contains(t, submit(true), "; packaging 1 file(s) (7 B uncompressed) from the working tree.") + reuse := submit(true) + assert.Contains(t, reuse, "No git repository at "+filepath.ToSlash(root)+"; reusing the cached snapshot (1 file(s), 7 B uncompressed).\n") + assert.NotContains(t, reuse, "packaging") + assert.Len(t, tarballs(), 1, "the resubmit must not upload again") + + assert.NotContains(t, submit(false), "No git repository") + + require.NoError(t, os.MkdirAll(filepath.Join(root, "lib"), 0o755)) + runGit(t, filepath.Join(root, "lib"), "init", "-q") + writeRepoFile(t, root, "lib/x.py", "x") + jsonOut := submit(false) + assert.Contains(t, jsonOut, "Warning: the snapshot leaves out 1 listed path(s)") + assert.NotContains(t, jsonOut, "No git repository") +} + +// A git repository gets no notice: git scopes its snapshot. +func TestSubmitWorkloadGitRepoNoNotice(t *testing.T) { + w, _ := nonGitSubmitServer(t) + repo := newTestRepo(t) + writeRepoFile(t, repo, "train.py", "print()") + commitAll(t, repo, "init") + cfg, cfgPath := nonGitSnapshotConfig(t, repo) + + var stderr syncBuffer + ctx := cmdio.InContext(t.Context(), cmdio.NewIO(t.Context(), flags.OutputText, strings.NewReader(""), io.Discard, &stderr, "", "")) + _, _, err := submitWorkload(ctx, w, cfg, cfgPath, "idem", true) + require.NoError(t, err) + assert.NotContains(t, stderr.String(), "No git repository") +} diff --git a/cmd/air/snapshot_upload.go b/cmd/air/snapshot_upload.go index 209139c4055..04ab179c8da 100644 --- a/cmd/air/snapshot_upload.go +++ b/cmd/air/snapshot_upload.go @@ -15,6 +15,7 @@ import ( "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/filer" "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/tarpack" "github.com/databricks/databricks-sdk-go" ) @@ -24,14 +25,115 @@ import ( // latency to AIR submissions. const uploadProvenanceSidecars = false +// snapshotLargeSourceBytes is the uncompressed size of the shipped files at or +// above which the non-git snapshot notice suggests narrowing the selection. The +// value is an arbitrary default, not derived from measurements; tune it freely. +const snapshotLargeSourceBytes = 64 << 20 + +// snapshotSkippedPathsShown caps how many skipped paths the warning names. +const snapshotSkippedPathsShown = 5 + +// humanBytes formats a byte count using binary (1024-based) units. It is a copy +// of humanBytes in cmd/fs/upload_progress.go, which is unexported there. +func humanBytes(n int64) string { + const unit = 1024 + if n < unit { + return fmt.Sprintf("%d B", n) + } + units := []string{"KiB", "MiB", "GiB", "TiB", "PiB"} + div, exp := int64(unit), 0 + for n/div >= unit && exp < len(units)-1 { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %s", float64(n)/float64(div), units[exp]) +} + +// shippedSnapshotFiles applies the plain tarball's entry rule (tarpack.Archives) +// to a listing: it returns how many entries the tarball will hold, their +// uncompressed content bytes (symlinks carry none), and the listed paths it will +// skip. ls-files lists a nested git repository as one directory entry, so that +// repository's files are skipped with it. +func shippedSnapshotFiles(files []snapshotFile) (count int, sourceBytes int64, skipped []string) { + for _, f := range files { + mode := fs.FileMode(f.mode) + if !tarpack.Archives(mode) { + skipped = append(skipped, filepath.ToSlash(f.rel)) + continue + } + count++ + if mode.IsRegular() { + sourceBytes += f.size + } + } + return count, sourceBytes, skipped +} + +// plainTarNoticeLines renders the non-git snapshot disclosure. The size is the +// uncompressed shipped bytes, which include_paths and .gitignore control; the +// compressed tarball size stays telemetry-only. reuse marks a cache hit, where +// nothing is packaged or uploaded, so the size tip is omitted. +func plainTarNoticeLines(repoPath string, count int, sourceBytes int64, reuse bool) []string { + base := filepath.ToSlash(repoPath) + total := humanBytes(sourceBytes) + if reuse { + return []string{fmt.Sprintf("No git repository at %s; reusing the cached snapshot (%d file(s), %s uncompressed).", base, count, total)} + } + lines := []string{fmt.Sprintf("No git repository at %s; packaging %d file(s) (%s uncompressed) from the working tree.", base, count, total)} + if sourceBytes >= snapshotLargeSourceBytes { + lines = append(lines, "Tip: limit the snapshot with code_source.include_paths or a .gitignore in the directory.") + } + return lines +} + +// skippedSnapshotWarning names up to snapshotSkippedPathsShown listed paths that +// the tarball leaves out. +func skippedSnapshotWarning(skipped []string) string { + shown := skipped + more := "" + if len(shown) > snapshotSkippedPathsShown { + shown = shown[:snapshotSkippedPathsShown] + more = fmt.Sprintf(" (and %d more)", len(skipped)-snapshotSkippedPathsShown) + } + return fmt.Sprintf("Warning: the snapshot leaves out %d listed path(s) that are not regular files or symlinks, such as nested git repositories: %s%s.", len(skipped), strings.Join(shown, ", "), more) +} + +// preparedSnapshot is a code_source snapshot resolved up to, but not including, +// packaging: the plan, the content-addressed tarball name, the plain_tar listing +// and whether the store already holds the tarball. submitWorkload resolves it +// before the staging spinner starts so the snapshot notice prints first. +type preparedSnapshot struct { + repoPath string + plan snapshotPlan + store filer.Filer + tarName string + remote string + // files is the plain_tar listing; nil for git_archive. + files []snapshotFile + exists bool + // result carries the telemetry fields known before packaging, and is + // returned on a resolution failure after the plan is known. + result snapshotResult +} + // uploadSnapshot packages the code_source into a tarball and uploads it to the // artifact store's .internal directory, returning the remote path to attach to the // ai_runtime_task. Uploading reuses libs/filer for the workspace/volume write without // depending on the bundle package (see uploadSnapshotTarball / snapshotUploadFiler). func uploadSnapshot(ctx context.Context, w *databricks.WorkspaceClient, snap *snapshotSourceConfig, configPath, snapshotArtifactPath string, sidecarStore filer.Filer, sidecarBase string) (snapshotResult, error) { + p, err := prepareSnapshot(ctx, w, snap, configPath, snapshotArtifactPath) + if err != nil { + return p.result, err + } + return p.upload(ctx, sidecarStore, sidecarBase) +} + +// prepareSnapshot resolves the root path, packaging plan and tarball name, and +// checks whether the tarball is already uploaded. +func prepareSnapshot(ctx context.Context, w *databricks.WorkspaceClient, snap *snapshotSourceConfig, configPath, snapshotArtifactPath string) (preparedSnapshot, error) { repoPath, err := resolveRootPath(ctx, snap.RootPath, filepath.Dir(configPath)) if err != nil { - return snapshotResult{}, err + return preparedSnapshot{}, err } // Resolve how to package before touching the tarball: git_archive (pinned commit) vs @@ -39,7 +141,7 @@ func uploadSnapshot(ctx context.Context, w *databricks.WorkspaceClient, snap *sn // the already-uploaded tarball instead of re-packaging and re-uploading. plan, err := resolveSnapshotPlan(ctx, newGitRepo(repoPath), snap.Git, snap.IncludePaths) if err != nil { - return snapshotResult{}, err + return preparedSnapshot{}, err } // remote_volume, when set, is a UC Volume path; snapshotUploadFiler routes @@ -47,7 +149,33 @@ func uploadSnapshot(ctx context.Context, w *databricks.WorkspaceClient, snap *sn if snap.RemoteVolume != nil { snapshotArtifactPath = *snap.RemoteVolume } - result, err := uploadSnapshotTarball(ctx, w, repoPath, plan, snapshotArtifactPath) + return prepareSnapshotTarball(ctx, w, repoPath, plan, snapshotArtifactPath) +} + +// announceSnapshot tells the user what a non-git snapshot ships. The skipped-path +// warning prints in every output mode, since those contents are silently missing +// from the run; the notice is progress output and prints only when showNotice is +// set (text mode). Git snapshots are scoped by git and get neither. +func announceSnapshot(ctx context.Context, p preparedSnapshot, showNotice bool) { + if p.plan.mode != modePlainTar || p.plan.isGitRepo { + return + } + count, sourceBytes, skipped := shippedSnapshotFiles(p.files) + if len(skipped) > 0 { + cmdio.LogString(ctx, skippedSnapshotWarning(skipped)) + } + if !showNotice { + return + } + for _, line := range plainTarNoticeLines(p.repoPath, count, sourceBytes, p.exists) { + cmdio.LogProgress(ctx, line) + } +} + +// upload packages and uploads the prepared snapshot (unless it is already in the +// store) and returns its remote code_source_path. +func (p preparedSnapshot) upload(ctx context.Context, sidecarStore filer.Filer, sidecarBase string) (snapshotResult, error) { + result, err := p.uploadTarball(ctx) if err != nil { return result, err } @@ -65,8 +193,8 @@ func uploadSnapshot(ctx context.Context, w *databricks.WorkspaceClient, snap *sn // cache) or serve a prior run's stale provenance on a cache hit. They also live in // the per-run launch dir, // not the shared artifact dir, so they don't accumulate. Keep them out of the tar. - if uploadProvenanceSidecars && plan.isGitRepo { - result.GitStatePath, result.GitDiffPath = uploadSnapshotSidecars(ctx, sidecarStore, sidecarBase, newGitRepo(repoPath), plan) + if uploadProvenanceSidecars && p.plan.isGitRepo { + result.GitStatePath, result.GitDiffPath = uploadSnapshotSidecars(ctx, sidecarStore, sidecarBase, newGitRepo(p.repoPath), p.plan) } return result, nil } @@ -133,7 +261,11 @@ func countSnapshotFiles(ctx context.Context, snap *snapshotSourceConfig, configP return git.countFilesAtCommit(ctx, plan.commitSHA, plan.subtreePrefix, plan.includePaths) } files, err := snapshotFiles(ctx, repoPath, plan.includePaths, plan.isGitRepo) - return len(files), err + if err != nil { + return 0, err + } + count, _, _ := shippedSnapshotFiles(files) + return count, nil } // snapshotTarName resolves the content-addressed upload filename for the snapshot and, @@ -172,32 +304,49 @@ const airArtifactInternalDir = ".internal" // uploadSnapshotTarball uploads the packaged snapshot to /.internal and returns // the remote code_source_path to attach to the ai_runtime_task. artifactPath is either // the user's repo_snapshots directory or a configured UC Volume. -// -// The tarball name is content-addressed — by (commit, include_paths, root_path subtree) -// for git_archive and by the working-tree metadata fingerprint for plain_tar — so if -// the identical object already exists we skip packaging and upload entirely and reuse it. func uploadSnapshotTarball(ctx context.Context, w *databricks.WorkspaceClient, repoPath string, plan snapshotPlan, artifactPath string) (snapshotResult, error) { - result := snapshotResult{UsesGit: &plan.isGitRepo, PackagingMode: &plan.mode} + p, err := prepareSnapshotTarball(ctx, w, repoPath, plan, artifactPath) + if err != nil { + return p.result, err + } + return p.uploadTarball(ctx) +} + +// prepareSnapshotTarball resolves the upload store and the content-addressed tarball +// name — by (commit, include_paths, root_path subtree) for git_archive and by the +// working-tree metadata fingerprint for plain_tar — and checks whether the identical +// object already exists, in which case uploadTarball skips packaging and upload. +func prepareSnapshotTarball(ctx context.Context, w *databricks.WorkspaceClient, repoPath string, plan snapshotPlan, artifactPath string) (preparedSnapshot, error) { + p := preparedSnapshot{ + repoPath: repoPath, + plan: plan, + result: snapshotResult{UsesGit: &plan.isGitRepo, PackagingMode: &plan.mode}, + } f, uploadPath, err := snapshotUploadFiler(ctx, w, artifactPath) if err != nil { - return result, err + return p, err } + p.store = f - tarName, files, err := snapshotTarName(ctx, repoPath, plan) + p.tarName, p.files, err = snapshotTarName(ctx, repoPath, plan) if err != nil { - return result, err + return p, err } // code_source_path is content-addressed by tarName, so it is the same whether we // upload the bytes now or reuse an object already in the store. - remote := path.Join(uploadPath, tarName) + p.remote = path.Join(uploadPath, p.tarName) - exists, err := snapshotExists(ctx, f, tarName) - if err != nil { - return result, err - } - if exists { - log.Debugf(ctx, "snapshot upload skipped; reusing %s", remote) - result.CodeSourcePath = remote + p.exists, err = snapshotExists(ctx, f, p.tarName) + return p, err +} + +// uploadTarball packages and uploads the prepared tarball, or reuses the stored one. +func (p preparedSnapshot) uploadTarball(ctx context.Context) (snapshotResult, error) { + result := p.result + tarName := p.tarName + if p.exists { + log.Debugf(ctx, "snapshot upload skipped; reusing %s", p.remote) + result.CodeSourcePath = p.remote result.PackagingDurationMs = new(int64(0)) result.UploadDurationMs = new(int64(0)) return result, nil @@ -211,7 +360,7 @@ func uploadSnapshotTarball(ctx context.Context, w *databricks.WorkspaceClient, r tarball := filepath.Join(tmp, tarName) packagingStart := time.Now() - err = packageSnapshot(ctx, repoPath, plan, files, tarball) + err = packageSnapshot(ctx, p.repoPath, p.plan, p.files, tarball) result.PackagingDurationMs = new(time.Since(packagingStart).Milliseconds()) if err != nil { return result, err @@ -231,12 +380,12 @@ func uploadSnapshotTarball(ctx context.Context, w *databricks.WorkspaceClient, r } cmdio.LogProgress(ctx, fmt.Sprintf("Uploading %s...", tarName)) uploadStart := time.Now() - err = f.Write(ctx, tarName, file, filer.OverwriteIfExists, filer.CreateParentDirectories) + err = p.store.Write(ctx, tarName, file, filer.OverwriteIfExists, filer.CreateParentDirectories) result.UploadDurationMs = new(time.Since(uploadStart).Milliseconds()) if err != nil { return result, fmt.Errorf("failed to upload snapshot %s: %w", tarName, err) } - result.CodeSourcePath = remote + result.CodeSourcePath = p.remote return result, nil } diff --git a/libs/tarpack/tarpack.go b/libs/tarpack/tarpack.go index 963988c0858..2139b3b0791 100644 --- a/libs/tarpack/tarpack.go +++ b/libs/tarpack/tarpack.go @@ -37,6 +37,12 @@ func Write(w io.Writer, entries []Entry) error { return tw.Close() } +// Archives reports whether [Write] stores an entry of this file mode: regular +// files and symlinks are archived, every other type is skipped. +func Archives(mode fs.FileMode) bool { + return mode.IsRegular() || mode&fs.ModeSymlink != 0 +} + func writeEntry(tw *tar.Writer, e Entry) error { // Lstat, not Stat, so a symlink is archived as a link rather than followed. info, err := os.Lstat(e.Path) @@ -44,7 +50,7 @@ func writeEntry(tw *tar.Writer, e Entry) error { return fmt.Errorf("stat %s: %w", e.Path, err) } mode := info.Mode() - if !mode.IsRegular() && mode&fs.ModeSymlink == 0 { + if !Archives(mode) { return nil }