Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 17 additions & 2 deletions cmd/air/runsubmit.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"fmt"
"net/http"
"path"
"slices"
"strconv"
"strings"
"time"
Expand Down Expand Up @@ -359,14 +360,27 @@ func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *run
// Package and upload the code snapshot, if any; the remote code_source_path
// rides the ai_runtime_task. A run with no code_source leaves it empty.
// Snapshot is the only code_source type.
// The snapshot format depends on the capabilities validation advertises. Only the format
// decision waits for validation: planning, listing and keying the snapshot run in parallel
// with it, and v1 submits do not wait longer than before.
validated := make(chan struct{})
var capabilities []string
var stageSnapshot func(context.Context) (snapshotResult, error)
if cfg.CodeSource != nil && cfg.CodeSource.Snapshot != nil {
// Default snapshot tarballs land in the user's shared repo_snapshots dir;
// uploadSnapshot replaces this when remote_volume is configured.
snapshotArtifactPath := path.Join(base, ".air", "repo_snapshots")
v2Advertised := func(ctx context.Context) (bool, error) {
select {
case <-validated:
case <-ctx.Done():
return false, ctx.Err()
}
return slices.Contains(capabilities, codeSnapshotV2Capability), nil
}
stageSnapshot = func(ctx context.Context) (snapshotResult, error) {
// Sidecars land in the run's launch dir (funcDir) via fc, next to command.sh.
return uploadSnapshot(ctx, w, cfg.CodeSource.Snapshot, configPath, snapshotArtifactPath, fc, funcDir)
return uploadSnapshot(ctx, w, cfg.CodeSource.Snapshot, configPath, snapshotArtifactPath, fc, funcDir, v2Advertised)
}
}

Expand All @@ -377,7 +391,8 @@ func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *run
group, stagingCtx := errgroup.WithContext(ctx)
var validationUnavailable bool
group.Go(func() (err error) {
validationUnavailable, err = preflightValidate(stagingCtx, w, cfg, commandPath, containers, token)
defer close(validated)
validationUnavailable, capabilities, err = preflightValidate(stagingCtx, w, cfg, commandPath, containers, token)
return err
})
group.Go(func() error {
Expand Down
10 changes: 5 additions & 5 deletions cmd/air/runsubmit_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -862,11 +862,11 @@ code_source:

ctx := cmdio.MockDiscard(t.Context())
sidecarStore, sidecarBase := testSidecarStore(t, w)
first, err := uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase)
first, err := uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase, fixedV2(false))
require.NoError(t, err)
require.NotZero(t, snapshotUploads.Load(), "first submit should upload the tarball")
afterFirst := snapshotUploads.Load()
second, err := uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase)
second, err := uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase, fixedV2(false))
require.NoError(t, err)

// Content-addressed name: not the bare dir, but a 16-hex-char fingerprint.
Expand Down Expand Up @@ -938,9 +938,9 @@ code_source:

ctx := cmdio.MockDiscard(t.Context())
sidecarStore, sidecarBase := testSidecarStore(t, w)
first, err := uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase)
first, err := uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase, fixedV2(false))
require.NoError(t, err)
second, err := uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase)
second, err := uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase, fixedV2(false))
require.NoError(t, err)

// Same pinned commit → identical content-addressed remote path, uploaded once
Expand Down Expand Up @@ -992,7 +992,7 @@ code_source:

ctx := cmdio.MockDiscard(t.Context())
sidecarStore, sidecarBase := testSidecarStore(t, w)
snap, err := uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase)
snap, err := uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase, fixedV2(false))
require.NoError(t, err)

assert.Empty(t, snap.GitStatePath)
Expand Down
18 changes: 15 additions & 3 deletions cmd/air/snapshot_upload.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ const uploadProvenanceSidecars = false
// artifact store's .internal directory, returning the remote path to attach to the
// ai_runtime_task. Uploading reuses libs/filer for the workspace/volume write without
// depending on the bundle package (see uploadSnapshotTarball / snapshotUploadFiler).
func uploadSnapshot(ctx context.Context, w *databricks.WorkspaceClient, snap *snapshotSourceConfig, configPath, snapshotArtifactPath string, sidecarStore filer.Filer, sidecarBase string) (snapshotResult, error) {
func uploadSnapshot(ctx context.Context, w *databricks.WorkspaceClient, snap *snapshotSourceConfig, configPath, snapshotArtifactPath string, sidecarStore filer.Filer, sidecarBase string, v2Advertised v2Decision) (snapshotResult, error) {
repoPath, err := resolveRootPath(ctx, snap.RootPath, filepath.Dir(configPath))
if err != nil {
return snapshotResult{}, err
Expand All @@ -50,7 +50,7 @@ func uploadSnapshot(ctx context.Context, w *databricks.WorkspaceClient, snap *sn
if snap.RemoteVolume != nil {
snapshotArtifactPath = *snap.RemoteVolume
}
result, err := uploadSnapshotTarball(ctx, w, repoPath, plan, snapshotArtifactPath)
result, err := uploadSnapshotTarball(ctx, w, repoPath, plan, snapshotArtifactPath, v2Advertised)
if err != nil {
return result, err
}
Expand Down Expand Up @@ -160,7 +160,7 @@ const airArtifactInternalDir = ".internal"
// The tarball name is content-addressed — by (commit, include_paths, root_path subtree)
// for git_archive and by the working-tree metadata fingerprint for plain_tar — so if
// the identical object already exists we skip packaging and upload entirely and reuse it.
func uploadSnapshotTarball(ctx context.Context, w *databricks.WorkspaceClient, repoPath string, plan snapshotPlan, artifactPath string) (snapshotResult, error) {
func uploadSnapshotTarball(ctx context.Context, w *databricks.WorkspaceClient, repoPath string, plan snapshotPlan, artifactPath string, v2Advertised v2Decision) (snapshotResult, error) {
result := snapshotResult{UsesGit: &plan.isGitRepo, PackagingMode: &plan.mode}
f, uploadPath, err := snapshotUploadFiler(ctx, w, artifactPath)
if err != nil {
Expand All @@ -171,6 +171,18 @@ func uploadSnapshotTarball(ctx context.Context, w *databricks.WorkspaceClient, r
if err != nil {
return result, err
}
// The format decision is the first point that needs validation. Only plain_tar can be v2, so
// git_archive never waits. The cache check below differs by format, so it comes after.
if plan.mode == modePlainTar {
v2, err := v2Advertised(ctx)
if err != nil {
return result, err
}
if v2 {
err := uploadSnapshotV2(ctx, f, uploadPath, repoPath, files, &result)
return result, err
}
}
// code_source_path is content-addressed by tarName, so it is the same whether we
// upload the bytes now or reuse an object already in the store.
remote := path.Join(uploadPath, tarName)
Expand Down
8 changes: 4 additions & 4 deletions cmd/air/snapshot_upload_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ func TestUploadSnapshotPlainTarWritesIndexAndTarball(t *testing.T) {

ctx := cmdio.MockDiscard(t.Context())
sidecarStore, sidecarBase := testSidecarStore(t, w)
first, err := uploadSnapshot(ctx, w, snap, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase)
first, err := uploadSnapshot(ctx, w, snap, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase, fixedV2(false))
require.NoError(t, err)

tarName := path.Base(first.CodeSourcePath)
Expand All @@ -97,7 +97,7 @@ func TestUploadSnapshotPlainTarWritesIndexAndTarball(t *testing.T) {
assert.Len(t, rec.index.Members, 1)

rec.reset()
second, err := uploadSnapshot(ctx, w, snap, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase)
second, err := uploadSnapshot(ctx, w, snap, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase, fixedV2(false))
require.NoError(t, err)
assert.Equal(t, first.CodeSourcePath, second.CodeSourcePath)
assert.Empty(t, rec.names(), "cache hit should write neither the index nor the tarball")
Expand All @@ -112,7 +112,7 @@ func TestUploadSnapshotPlainTarMissingSidecarReuploadsBoth(t *testing.T) {

ctx := cmdio.MockDiscard(t.Context())
sidecarStore, sidecarBase := testSidecarStore(t, w)
first, err := uploadSnapshot(ctx, w, snap, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase)
first, err := uploadSnapshot(ctx, w, snap, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase, fixedV2(false))
require.NoError(t, err)
tarName := path.Base(first.CodeSourcePath)

Expand All @@ -121,7 +121,7 @@ func TestUploadSnapshotPlainTarMissingSidecarReuploadsBoth(t *testing.T) {
require.NoError(t, uploadDir.Delete(ctx, tarName+segmentIndexSuffix))

rec.reset()
second, err := uploadSnapshot(ctx, w, snap, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase)
second, err := uploadSnapshot(ctx, w, snap, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase, fixedV2(false))
require.NoError(t, err)
assert.Equal(t, first.CodeSourcePath, second.CodeSourcePath)
assert.Equal(t, map[string]bool{tarName: true, tarName + segmentIndexSuffix: true}, rec.names())
Expand Down
Loading
Loading