Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/air/non-git-snapshot-notice.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* `air run` from a directory that is not a git repository now says so before packaging, with the number of files and their uncompressed size (or that it reuses the cached snapshot), suggests `code_source.include_paths` or a `.gitignore` when the files total 64 MiB or more, and warns about listed paths it leaves out, such as nested git repositories. `--dry-run` no longer counts those paths. The notice is not printed with `-o json`. ([#7618](https://github.com/databricks/cli/pull/7618))
2 changes: 2 additions & 0 deletions acceptance/air/run-submit-non-git/databricks.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
bundle:
name: air-run-submit-non-git
1 change: 1 addition & 0 deletions acceptance/air/run-submit-non-git/out.test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Cloud = false
38 changes: 38 additions & 0 deletions acceptance/air/run-submit-non-git/output.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@

=== submit from a non-git directory

>>> [CLI] air run -f run.yaml
Submitting experiment: submit-non-git-smoke
No git repository at [TEST_TMP_DIR]; packaging 5 file(s) (64.3 KiB uncompressed) from the working tree.
Uploading [SNAPSHOT_TARBALL]...
Submitted workload with Job Run ID: 555
View job run at: [DATABRICKS_URL]/jobs/runs/555?o=[WORKSPACE_ID]

Tip: use --watch when submitting a run to stream logs to your terminal.
Stream logs after submission using:
databricks air logs 555

=== unchanged resubmit reuses the cached snapshot

>>> [CLI] air run -f run.yaml
Submitting experiment: submit-non-git-smoke
No git repository at [TEST_TMP_DIR]; reusing the cached snapshot (5 file(s), 64.3 KiB uncompressed).
Submitted workload with Job Run ID: 555
View job run at: [DATABRICKS_URL]/jobs/runs/555?o=[WORKSPACE_ID]

Tip: use --watch when submitting a run to stream logs to your terminal.
Stream logs after submission using:
databricks air logs 555

=== json mode: no notice, like the Submitting line

>>> [CLI] -o json air run -f run.yaml
{
"v": 1,
"ts": "[TIMESTAMP]",
"data": {
"status": "PENDING",
"run_id": "555",
"dashboard_url": "[DATABRICKS_URL]/jobs/runs/555?o=[WORKSPACE_ID]"
}
}
9 changes: 9 additions & 0 deletions acceptance/air/run-submit-non-git/run.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
experiment_name: submit-non-git-smoke
command: python train.py
compute:
accelerator_type: GPU_1xH100
num_accelerators: 1
code_source:
type: snapshot
snapshot:
root_path: .
15 changes: 15 additions & 0 deletions acceptance/air/run-submit-non-git/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
# No git-repo-init here: the directory stays non-git, so the snapshot must fall
# back to plain_tar over everything in the working tree, including run.log and
# the binary file, and the CLI announces that before packaging.
echo "print('hello')" > main.py
echo "some training log output" > run.log
head -c 65536 /dev/zero > weights.bin

title "submit from a non-git directory"
trace $CLI air run -f run.yaml

title "unchanged resubmit reuses the cached snapshot"
trace $CLI air run -f run.yaml

title "json mode: no notice, like the Submitting line"
trace $CLI -o json air run -f run.yaml
37 changes: 37 additions & 0 deletions acceptance/air/run-submit-non-git/test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
# A real (non-dry-run) submit from a directory with no git repository: the
# snapshot falls back to plain_tar over the whole working tree, and the CLI
# says so (file count + uncompressed size) before packaging, including on the
# second, cache-hit submit. -o json skips the notice.
RecordRequests = true

# main.py/run.log/weights.bin are generated by the script (the point of the
# test is that a non-git code_source packages them); not committed inputs.
Ignore = ["main.py", "run.log", "weights.bin"]

# The SDK probes host reachability with a HEAD request; stub it for determinism.
[[Server]]
Pattern = "HEAD /"
Response.Body = ''

[[Server]]
Pattern = "POST /api/2.0/ai-training/config:validate"
Response.Body = '{}'

[[Server]]
Pattern = "POST /api/2.2/jobs/runs/submit"
Response.Body = '''
{"run_id": 555}
'''

# The snapshot tarball is named <dir>_<key[:16]>.tar.gz; the temp-dir basename
# and the working-tree fingerprint vary per run, so collapse the name. The
# notice's count and size are NOT masked: the committed inputs make them
# deterministic, so the golden asserts them exactly.
[[Repls]]
Old = '[a-zA-Z0-9._-]+_[0-9a-f]{16}\.tar\.gz'
New = '[SNAPSHOT_TARBALL]'

# The per-run launch directory ends in <experiment>_<16 hex>; the random suffix varies.
[[Repls]]
Old = 'submit-non-git-smoke_[0-9a-f]{16}'
New = 'submit-non-git-smoke_[RUN_ID]'
15 changes: 12 additions & 3 deletions cmd/air/runsubmit.go
Original file line number Diff line number Diff line change
Expand Up @@ -333,7 +333,7 @@ func stageRunArtifacts(ctx context.Context, launchWriter fileWriter, items []upl
// payload, and submit it. It
// returns the new run_id and its dashboard URL, which carries the ?o= workspace
// hint only when the workspace ID could be resolved. showProgress enables the stderr
// staging spinner (text mode only).
// staging spinner and the non-git snapshot notice (text mode only).
func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *runConfig, configPath, idempotencyKey string, showProgress bool) (runID int64, runURL string, err error) {
start := time.Now()
var snap snapshotResult
Expand Down Expand Up @@ -364,11 +364,20 @@ func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *run
var stageSnapshot func(context.Context) (snapshotResult, error)
if cfg.CodeSource != nil && cfg.CodeSource.Snapshot != nil {
// Default snapshot tarballs land in the user's shared repo_snapshots dir;
// uploadSnapshot replaces this when remote_volume is configured.
// prepareSnapshot replaces this when remote_volume is configured.
snapshotArtifactPath := path.Join(base, ".air", "repo_snapshots")
// Resolve the snapshot before staging so its notice prints before the
// staging spinner starts drawing on stderr.
var prepared preparedSnapshot
prepared, err = prepareSnapshot(ctx, w, cfg.CodeSource.Snapshot, configPath, snapshotArtifactPath)
if err != nil {
snap = prepared.result
return 0, "", err
}
announceSnapshot(ctx, prepared, showProgress)
stageSnapshot = func(ctx context.Context) (snapshotResult, error) {
// Sidecars land in the run's launch dir (funcDir) via fc, next to command.sh.
return uploadSnapshot(ctx, w, cfg.CodeSource.Snapshot, configPath, snapshotArtifactPath, fc, funcDir)
return prepared.upload(ctx, fc, funcDir)
}
}

Expand Down
3 changes: 3 additions & 0 deletions cmd/air/snapshot_git.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ import (
"os/exec"
"strings"
"time"

"github.com/databricks/cli/libs/log"
)

// Local, no-network git introspection and the git-state provenance sidecar, ported
Expand Down Expand Up @@ -68,6 +70,7 @@ func (g gitRepo) runTo(ctx context.Context, w io.Writer, args ...string) error {
func (g gitRepo) isRepository(ctx context.Context) bool {
out, err := g.run(ctx, "rev-parse", "--is-inside-work-tree")
if err != nil {
log.Debugf(ctx, "git repository probe for %s failed: %v", g.path, err)
return false
}
return strings.TrimSpace(out) == "true"
Expand Down
Loading
Loading