diff --git a/cmd/air/runsubmit.go b/cmd/air/runsubmit.go index f01198d2b7a..451772da098 100644 --- a/cmd/air/runsubmit.go +++ b/cmd/air/runsubmit.go @@ -8,6 +8,7 @@ import ( "fmt" "net/http" "path" + "slices" "strconv" "strings" "time" @@ -359,14 +360,27 @@ func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *run // Package and upload the code snapshot, if any; the remote code_source_path // rides the ai_runtime_task. A run with no code_source leaves it empty. // Snapshot is the only code_source type. + // The snapshot format depends on the capabilities validation advertises. Only the format + // decision waits for validation: planning, listing and keying the snapshot run in parallel + // with it, and v1 submits do not wait longer than before. + validated := make(chan struct{}) + var capabilities []string var stageSnapshot func(context.Context) (snapshotResult, error) if cfg.CodeSource != nil && cfg.CodeSource.Snapshot != nil { // Default snapshot tarballs land in the user's shared repo_snapshots dir; // uploadSnapshot replaces this when remote_volume is configured. snapshotArtifactPath := path.Join(base, ".air", "repo_snapshots") + v2Advertised := func(ctx context.Context) (bool, error) { + select { + case <-validated: + case <-ctx.Done(): + return false, ctx.Err() + } + return slices.Contains(capabilities, codeSnapshotV2Capability), nil + } stageSnapshot = func(ctx context.Context) (snapshotResult, error) { // Sidecars land in the run's launch dir (funcDir) via fc, next to command.sh. - return uploadSnapshot(ctx, w, cfg.CodeSource.Snapshot, configPath, snapshotArtifactPath, fc, funcDir) + return uploadSnapshot(ctx, w, cfg.CodeSource.Snapshot, configPath, snapshotArtifactPath, fc, funcDir, v2Advertised) } } @@ -377,7 +391,8 @@ func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *run group, stagingCtx := errgroup.WithContext(ctx) var validationUnavailable bool group.Go(func() (err error) { - validationUnavailable, err = preflightValidate(stagingCtx, w, cfg, commandPath, containers, token) + defer close(validated) + validationUnavailable, capabilities, err = preflightValidate(stagingCtx, w, cfg, commandPath, containers, token) return err }) group.Go(func() error { diff --git a/cmd/air/runsubmit_test.go b/cmd/air/runsubmit_test.go index e6eb46e4491..87116fde10e 100644 --- a/cmd/air/runsubmit_test.go +++ b/cmd/air/runsubmit_test.go @@ -862,11 +862,11 @@ code_source: ctx := cmdio.MockDiscard(t.Context()) sidecarStore, sidecarBase := testSidecarStore(t, w) - first, err := uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase) + first, err := uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase, fixedV2(false)) require.NoError(t, err) require.NotZero(t, snapshotUploads.Load(), "first submit should upload the tarball") afterFirst := snapshotUploads.Load() - second, err := uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase) + second, err := uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase, fixedV2(false)) require.NoError(t, err) // Content-addressed name: not the bare dir, but a 16-hex-char fingerprint. @@ -938,9 +938,9 @@ code_source: ctx := cmdio.MockDiscard(t.Context()) sidecarStore, sidecarBase := testSidecarStore(t, w) - first, err := uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase) + first, err := uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase, fixedV2(false)) require.NoError(t, err) - second, err := uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase) + second, err := uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase, fixedV2(false)) require.NoError(t, err) // Same pinned commit → identical content-addressed remote path, uploaded once @@ -992,7 +992,7 @@ code_source: ctx := cmdio.MockDiscard(t.Context()) sidecarStore, sidecarBase := testSidecarStore(t, w) - snap, err := uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase) + snap, err := uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase, fixedV2(false)) require.NoError(t, err) assert.Empty(t, snap.GitStatePath) diff --git a/cmd/air/snapshot_upload.go b/cmd/air/snapshot_upload.go index fd6903e2c75..c21b5b7c0c3 100644 --- a/cmd/air/snapshot_upload.go +++ b/cmd/air/snapshot_upload.go @@ -31,7 +31,7 @@ const uploadProvenanceSidecars = false // artifact store's .internal directory, returning the remote path to attach to the // ai_runtime_task. Uploading reuses libs/filer for the workspace/volume write without // depending on the bundle package (see uploadSnapshotTarball / snapshotUploadFiler). -func uploadSnapshot(ctx context.Context, w *databricks.WorkspaceClient, snap *snapshotSourceConfig, configPath, snapshotArtifactPath string, sidecarStore filer.Filer, sidecarBase string) (snapshotResult, error) { +func uploadSnapshot(ctx context.Context, w *databricks.WorkspaceClient, snap *snapshotSourceConfig, configPath, snapshotArtifactPath string, sidecarStore filer.Filer, sidecarBase string, v2Advertised v2Decision) (snapshotResult, error) { repoPath, err := resolveRootPath(ctx, snap.RootPath, filepath.Dir(configPath)) if err != nil { return snapshotResult{}, err @@ -50,7 +50,7 @@ func uploadSnapshot(ctx context.Context, w *databricks.WorkspaceClient, snap *sn if snap.RemoteVolume != nil { snapshotArtifactPath = *snap.RemoteVolume } - result, err := uploadSnapshotTarball(ctx, w, repoPath, plan, snapshotArtifactPath) + result, err := uploadSnapshotTarball(ctx, w, repoPath, plan, snapshotArtifactPath, v2Advertised) if err != nil { return result, err } @@ -160,7 +160,7 @@ const airArtifactInternalDir = ".internal" // The tarball name is content-addressed — by (commit, include_paths, root_path subtree) // for git_archive and by the working-tree metadata fingerprint for plain_tar — so if // the identical object already exists we skip packaging and upload entirely and reuse it. -func uploadSnapshotTarball(ctx context.Context, w *databricks.WorkspaceClient, repoPath string, plan snapshotPlan, artifactPath string) (snapshotResult, error) { +func uploadSnapshotTarball(ctx context.Context, w *databricks.WorkspaceClient, repoPath string, plan snapshotPlan, artifactPath string, v2Advertised v2Decision) (snapshotResult, error) { result := snapshotResult{UsesGit: &plan.isGitRepo, PackagingMode: &plan.mode} f, uploadPath, err := snapshotUploadFiler(ctx, w, artifactPath) if err != nil { @@ -171,6 +171,18 @@ func uploadSnapshotTarball(ctx context.Context, w *databricks.WorkspaceClient, r if err != nil { return result, err } + // The format decision is the first point that needs validation. Only plain_tar can be v2, so + // git_archive never waits. The cache check below differs by format, so it comes after. + if plan.mode == modePlainTar { + v2, err := v2Advertised(ctx) + if err != nil { + return result, err + } + if v2 { + err := uploadSnapshotV2(ctx, f, uploadPath, repoPath, files, &result) + return result, err + } + } // code_source_path is content-addressed by tarName, so it is the same whether we // upload the bytes now or reuse an object already in the store. remote := path.Join(uploadPath, tarName) diff --git a/cmd/air/snapshot_upload_test.go b/cmd/air/snapshot_upload_test.go index 3c6c7af64f8..4f9eb6695df 100644 --- a/cmd/air/snapshot_upload_test.go +++ b/cmd/air/snapshot_upload_test.go @@ -85,7 +85,7 @@ func TestUploadSnapshotPlainTarWritesIndexAndTarball(t *testing.T) { ctx := cmdio.MockDiscard(t.Context()) sidecarStore, sidecarBase := testSidecarStore(t, w) - first, err := uploadSnapshot(ctx, w, snap, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase) + first, err := uploadSnapshot(ctx, w, snap, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase, fixedV2(false)) require.NoError(t, err) tarName := path.Base(first.CodeSourcePath) @@ -97,7 +97,7 @@ func TestUploadSnapshotPlainTarWritesIndexAndTarball(t *testing.T) { assert.Len(t, rec.index.Members, 1) rec.reset() - second, err := uploadSnapshot(ctx, w, snap, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase) + second, err := uploadSnapshot(ctx, w, snap, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase, fixedV2(false)) require.NoError(t, err) assert.Equal(t, first.CodeSourcePath, second.CodeSourcePath) assert.Empty(t, rec.names(), "cache hit should write neither the index nor the tarball") @@ -112,7 +112,7 @@ func TestUploadSnapshotPlainTarMissingSidecarReuploadsBoth(t *testing.T) { ctx := cmdio.MockDiscard(t.Context()) sidecarStore, sidecarBase := testSidecarStore(t, w) - first, err := uploadSnapshot(ctx, w, snap, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase) + first, err := uploadSnapshot(ctx, w, snap, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase, fixedV2(false)) require.NoError(t, err) tarName := path.Base(first.CodeSourcePath) @@ -121,7 +121,7 @@ func TestUploadSnapshotPlainTarMissingSidecarReuploadsBoth(t *testing.T) { require.NoError(t, uploadDir.Delete(ctx, tarName+segmentIndexSuffix)) rec.reset() - second, err := uploadSnapshot(ctx, w, snap, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase) + second, err := uploadSnapshot(ctx, w, snap, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase, fixedV2(false)) require.NoError(t, err) assert.Equal(t, first.CodeSourcePath, second.CodeSourcePath) assert.Equal(t, map[string]bool{tarName: true, tarName + segmentIndexSuffix: true}, rec.names()) diff --git a/cmd/air/snapshot_v2.go b/cmd/air/snapshot_v2.go new file mode 100644 index 00000000000..1026f7091ea --- /dev/null +++ b/cmd/air/snapshot_v2.go @@ -0,0 +1,405 @@ +package aircmd + +// Snapshot format v2 stores a plain_tar working tree as content-addressed members and a +// manifest, so an iteration re-uploads only the members whose files changed. The launcher +// side reads the manifest; the CLI emits v2 only when the backend advertises +// codeSnapshotV2Capability, and otherwise keeps writing v1 (see snapshot_upload.go). + +import ( + "archive/tar" + "bytes" + "context" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path" + "path/filepath" + "runtime" + "slices" + "strings" + "time" + + "github.com/databricks/cli/libs/filer" + "github.com/databricks/cli/libs/log" + "golang.org/x/sync/errgroup" +) + +// codeSnapshotV2Capability is the config:validate capability that enables v2. +const codeSnapshotV2Capability = "code_snapshot_v2" + +// v2Decision reports whether the backend advertised v2. It blocks until config:validate has +// finished, so it is only called where the answer is needed. +type v2Decision func(ctx context.Context) (bool, error) + +const ( + v2Buckets = 16 + v2ManifestFormat = "air-snapshot-manifest" + v2ManifestVersion = 2 + v2ManifestSuffix = ".air.json" + v2ObjectsDir = "objects" + v2UploadConcurrency = 8 + v2UploadAttempts = 3 + v2UploadBackoff = 200 * time.Millisecond + v2KStepBytes = 16 << 20 + v2K16StepBytes = 128 << 20 + v2EndTarBytes = 2 * tarBlockSize +) + +// v2Manifest is the object at code_source_path. Members are in archive order and end with +// the end-of-archive member, so concatenating their gzip objects gives a valid v1 .tar.gz. +type v2Manifest struct { + Format string `json:"format"` + Version int `json:"version"` + Component string `json:"component"` + K int `json:"k"` + TarSize int64 `json:"tar_size"` + Members []v2Member `json:"members"` +} + +// v2Member is one object: the first 128 bits of sha256 over its uncompressed tar fragment. +type v2Member struct { + ID string `json:"id"` + GzSize int64 `json:"gz_size"` + TarSize int64 `json:"tar_size"` + + fragment string // local uncompressed fragment, not part of the manifest +} + +// v2Bucket assigns a file to one of v2Buckets by the hash of its path, so a file keeps its +// bucket across snapshots. +func v2Bucket(rel string) int { + sum := sha256.Sum256([]byte(rel)) + return int(binary.BigEndian.Uint64(sum[:8]) % v2Buckets) +} + +// kForTarSize returns how many members a snapshot of tarSize bytes uses, never fewer than +// prevK so that a shrinking repo keeps its members. +func kForTarSize(tarSize int64, prevK int) int { + k := 1 + switch { + case tarSize >= v2K16StepBytes: + k = 16 + case tarSize >= v2KStepBytes: + k = 4 + } + return max(k, prevK) +} + +// memberOf maps a bucket to one of k members. Buckets merge evenly, so each member of a +// k=4 snapshot holds whole buckets of the k=16 layout. +func memberOf(bucket, k int) int { + return bucket / (v2Buckets / k) +} + +// v2Header is the tar header for info. It carries the same owner fields as v1, so extraction +// restores the same ownership. Ids therefore depend on the machine that packaged the tree, +// and another machine re-uploads once. +func v2Header(info fs.FileInfo, name, linkTarget string) (*tar.Header, error) { + hdr, err := tar.FileInfoHeader(info, linkTarget) + if err != nil { + return nil, err + } + hdr.Name = name + return hdr, nil +} + +// writeV2Fragment writes the tar entries for files to fragmentPath and returns the member id +// and the fragment's size. The fragment has complete entries only: no end-of-archive marker +// and no record padding, so it can be concatenated with other fragments. +func writeV2Fragment(fragmentPath, repoPath, component string, files []snapshotFile) (string, int64, error) { + f, err := os.Create(fragmentPath) + if err != nil { + return "", 0, err + } + defer f.Close() + + hash := sha256.New() + tw := tar.NewWriter(io.MultiWriter(f, hash)) + for _, file := range files { + if err := writeV2Entry(tw, repoPath, component, file); err != nil { + return "", 0, err + } + } + if err := tw.Flush(); err != nil { + return "", 0, fmt.Errorf("failed to write snapshot member: %w", err) + } + info, err := f.Stat() + if err != nil { + return "", 0, err + } + return hex.EncodeToString(hash.Sum(nil)[:16]), info.Size(), nil +} + +// writeV2Entry adds one file to the tar stream, matching tarpack's rules: regular files and +// symlinks are archived, and other file types are skipped. +func writeV2Entry(tw *tar.Writer, repoPath, component string, file snapshotFile) error { + name := filepath.Join(repoPath, file.rel) + info, err := os.Lstat(name) + if err != nil { + return fmt.Errorf("stat %s: %w", name, err) + } + mode := info.Mode() + if !mode.IsRegular() && mode&fs.ModeSymlink == 0 { + return nil + } + link := "" + if mode&fs.ModeSymlink != 0 { + if link, err = os.Readlink(name); err != nil { + return fmt.Errorf("readlink %s: %w", name, err) + } + } + hdr, err := v2Header(info, path.Join(component, filepath.ToSlash(file.rel)), link) + if err != nil { + return fmt.Errorf("tar header for %s: %w", name, err) + } + if err := tw.WriteHeader(hdr); err != nil { + return fmt.Errorf("write header for %s: %w", name, err) + } + if !mode.IsRegular() { + return nil + } + f, err := os.Open(name) + if err != nil { + return fmt.Errorf("open %s: %w", name, err) + } + defer f.Close() + if _, err := io.Copy(tw, f); err != nil { + return fmt.Errorf("write %s: %w", name, err) + } + return nil +} + +// buildV2Members groups files into k members and writes one fragment per non-empty member +// in tmp. Members are returned in index order. Empty members are omitted because a gzip +// member holding no tar entries is not a valid tar archive. +func buildV2Members(ctx context.Context, tmp, repoPath, component string, files []snapshotFile, k int) ([]v2Member, error) { + groups := make([][]snapshotFile, k) + for _, file := range files { + m := memberOf(v2Bucket(filepath.ToSlash(file.rel)), k) + groups[m] = append(groups[m], file) + } + for _, group := range groups { + slices.SortFunc(group, func(a, b snapshotFile) int { + return strings.Compare(filepath.ToSlash(a.rel), filepath.ToSlash(b.rel)) + }) + } + + members := make([]v2Member, k) + var g errgroup.Group + g.SetLimit(max(1, min(k, runtime.NumCPU()))) + for i := range k { + g.Go(func() error { + fragment := filepath.Join(tmp, fmt.Sprintf("fragment-%d.tar", i)) + id, size, err := writeV2Fragment(fragment, repoPath, component, groups[i]) + if err != nil { + return err + } + members[i] = v2Member{ID: id, TarSize: size, fragment: fragment} + return nil + }) + } + if err := g.Wait(); err != nil { + return nil, err + } + return slices.DeleteFunc(members, func(m v2Member) bool { return m.TarSize == 0 }), nil +} + +// writeV2EndFragment writes the constant end-of-archive member: two zero blocks. +func writeV2EndFragment(tmp string) (v2Member, error) { + fragment := filepath.Join(tmp, "end.tar") + if err := os.WriteFile(fragment, make([]byte, v2EndTarBytes), 0o644); err != nil { + return v2Member{}, err + } + sum := sha256.Sum256(make([]byte, v2EndTarBytes)) + return v2Member{ID: hex.EncodeToString(sum[:16]), TarSize: v2EndTarBytes, fragment: fragment}, nil +} + +// v2ObjectName is the object path of a member, relative to the snapshot directory. +func v2ObjectName(id string) string { + return path.Join(v2ObjectsDir, id+".tar.gz") +} + +// putV2Members makes every member an object in store and records its gzip size. An object +// that already exists is only sized: its id is its content, so it is never rewritten. At most +// v2UploadConcurrency members are compressed or uploaded at once. +func putV2Members(ctx context.Context, store filer.Filer, members []v2Member, tmp string) error { + g, gctx := errgroup.WithContext(ctx) + g.SetLimit(v2UploadConcurrency) + for i := range members { + g.Go(func() error { + return putV2Member(gctx, store, &members[i], tmp) + }) + } + return g.Wait() +} + +// putV2Member writes one member as an object unless it already exists. +func putV2Member(ctx context.Context, store filer.Filer, m *v2Member, tmp string) error { + name := v2ObjectName(m.ID) + info, err := store.Stat(ctx, name) + switch { + case err == nil: + m.GzSize = info.Size() + return nil + case !errors.Is(err, fs.ErrNotExist): + return fmt.Errorf("failed to check snapshot object %s: %w", name, err) + } + + gzPath := filepath.Join(tmp, m.ID+".tar.gz") + if err := compressRaw(ctx, m.fragment, gzPath); err != nil { + return fmt.Errorf("failed to compress snapshot member %s: %w", m.ID, err) + } + info, err = os.Stat(gzPath) + if err != nil { + return err + } + m.GzSize = info.Size() + return withV2Retry(ctx, func() error { + f, err := os.Open(gzPath) + if err != nil { + return err + } + defer f.Close() + return store.Write(ctx, name, f, filer.OverwriteIfExists, filer.CreateParentDirectories) + }) +} + +// withV2Retry runs op up to v2UploadAttempts times with exponential backoff. Context +// cancellation is never retried. +func withV2Retry(ctx context.Context, op func() error) error { + var err error + for attempt := range v2UploadAttempts { + if err = op(); err == nil { + return nil + } + if ctx.Err() != nil || attempt == v2UploadAttempts-1 { + break + } + log.Debugf(ctx, "snapshot upload attempt %d failed, retrying: %v", attempt+1, err) + select { + case <-time.After(v2UploadBackoff << attempt): + case <-ctx.Done(): + return ctx.Err() + } + } + return err +} + +// previousV2K returns the member count of the newest manifest for component in store, or 0 +// when there is none. K only ratchets up, so the newest manifest is enough. +func previousV2K(ctx context.Context, store filer.Filer, component string) (int, error) { + entries, err := store.ReadDir(ctx, ".") + if errors.Is(err, fs.ErrNotExist) { + return 0, nil + } + if err != nil { + return 0, fmt.Errorf("failed to list snapshots: %w", err) + } + var newest fs.DirEntry + var newestTime time.Time + for _, entry := range entries { + name := entry.Name() + if !strings.HasPrefix(name, component+"_") || !strings.HasSuffix(name, v2ManifestSuffix) { + continue + } + info, err := entry.Info() + if err != nil { + continue + } + if newest == nil || info.ModTime().After(newestTime) { + newest, newestTime = entry, info.ModTime() + } + } + if newest == nil { + return 0, nil + } + rc, err := store.Read(ctx, newest.Name()) + if err != nil { + return 0, fmt.Errorf("failed to read snapshot manifest %s: %w", newest.Name(), err) + } + defer rc.Close() + var manifest v2Manifest + if err := json.NewDecoder(rc).Decode(&manifest); err != nil { + return 0, fmt.Errorf("failed to parse snapshot manifest %s: %w", newest.Name(), err) + } + return manifest.K, nil +} + +// uploadSnapshotV2 writes the working-tree snapshot in v2 form and sets result from it. The +// manifest is written last, after every object succeeded, so a visible manifest always refers +// to complete objects. An unchanged snapshot is found by its manifest name alone. +func uploadSnapshotV2(ctx context.Context, store filer.Filer, uploadPath, repoPath string, files []snapshotFile, result *snapshotResult) error { + component := filepath.Base(repoPath) + manifestName := component + "_" + computePlainTarKey(files)[:16] + v2ManifestSuffix + remote := path.Join(uploadPath, manifestName) + + cached, err := snapshotExists(ctx, store, manifestName) + if err != nil { + return err + } + if cached { + log.Debugf(ctx, "snapshot upload skipped; reusing %s", remote) + result.CodeSourcePath = remote + result.PackagingDurationMs = new(int64(0)) + result.UploadDurationMs = new(int64(0)) + return nil + } + + prevK, err := previousV2K(ctx, store, component) + if err != nil { + return err + } + tarTotal := int64(v2EndTarBytes) + for _, file := range files { + tarTotal += file.tarSize() + } + k := kForTarSize(tarTotal, prevK) + + tmp, err := os.MkdirTemp("", "air-snapshot-v2-*") + if err != nil { + return err + } + defer os.RemoveAll(tmp) + + packagingStart := time.Now() + members, err := buildV2Members(ctx, tmp, repoPath, component, files, k) + if err != nil { + return err + } + end, err := writeV2EndFragment(tmp) + if err != nil { + return err + } + members = append(members, end) + result.PackagingDurationMs = new(time.Since(packagingStart).Milliseconds()) + + uploadStart := time.Now() + if err := putV2Members(ctx, store, members, tmp); err != nil { + return err + } + manifest := v2Manifest{Format: v2ManifestFormat, Version: v2ManifestVersion, Component: component, K: k, Members: members} + var gzTotal int64 + for _, m := range members { + manifest.TarSize += m.TarSize + gzTotal += m.GzSize + } + data, err := json.Marshal(manifest) + if err != nil { + return fmt.Errorf("failed to encode snapshot manifest: %w", err) + } + if err := withV2Retry(ctx, func() error { + return store.Write(ctx, manifestName, bytes.NewReader(data), filer.OverwriteIfExists, filer.CreateParentDirectories) + }); err != nil { + return fmt.Errorf("failed to upload snapshot manifest %s: %w", manifestName, err) + } + result.UploadDurationMs = new(time.Since(uploadStart).Milliseconds()) + result.SizeBytes = &gzTotal + result.CodeSourcePath = remote + return nil +} diff --git a/cmd/air/snapshot_v2_test.go b/cmd/air/snapshot_v2_test.go new file mode 100644 index 00000000000..4ca4e15986a --- /dev/null +++ b/cmd/air/snapshot_v2_test.go @@ -0,0 +1,674 @@ +package aircmd + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "maps" + "net/http" + "os" + "os/exec" + "path" + "path/filepath" + "slices" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/filer" + "github.com/databricks/cli/libs/testserver" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fixedMtime pins file times so fragment bytes, and therefore ids, do not depend on when the +// test ran. +var fixedMtime = time.Unix(1_700_000_000, 0) + +// writeV2Tree writes files under a fresh repo directory with fixed modes and mtimes. The +// directory name is "code". +func writeV2Tree(t *testing.T, files map[string]string) string { + t.Helper() + repo := filepath.Join(t.TempDir(), "code") + for rel, content := range files { + writeRepoFile(t, repo, rel, content) + full := filepath.Join(repo, filepath.FromSlash(rel)) + require.NoError(t, os.Chmod(full, 0o644)) + require.NoError(t, os.Chtimes(full, fixedMtime, fixedMtime)) + } + return repo +} + +// v2TestTree is a repo with enough files to spread over several buckets. +func v2TestTree(n int) map[string]string { + files := make(map[string]string, n) + for i := range n { + files[fmt.Sprintf("pkg%d/mod%d.py", i%4, i)] = strings.Repeat(fmt.Sprintf("x = %d\n", i), 10+i) + } + return files +} + +// v2IDs returns the member ids of the repo's snapshot packed with k members, as a set. +func v2IDs(t *testing.T, repo string, k int) map[string]bool { + t.Helper() + files, err := snapshotFiles(t.Context(), repo, nil, false) + require.NoError(t, err) + members, err := buildV2Members(t.Context(), t.TempDir(), repo, filepath.Base(repo), files, k) + require.NoError(t, err) + ids := make(map[string]bool, len(members)) + for _, m := range members { + ids[m.ID] = true + } + return ids +} + +// idChanges returns how many ids in after are new, and how many in before are gone. A member +// whose content changed contributes one of each. +func idChanges(before, after map[string]bool) (added, removed int) { + for id := range after { + if !before[id] { + added++ + } + } + for id := range before { + if !after[id] { + removed++ + } + } + return added, removed +} + +func TestKForTarSize(t *testing.T) { + tests := []struct { + name string + tarSize int64 + prevK int + want int + }{ + {name: "small repo is one member", tarSize: 1 << 20, want: 1}, + {name: "16 MiB steps to four", tarSize: 16 << 20, want: 4}, + {name: "just under 128 MiB stays four", tarSize: 128<<20 - 1, want: 4}, + {name: "128 MiB steps to sixteen", tarSize: 128 << 20, want: 16}, + {name: "shrinking keeps previous k", tarSize: 1 << 20, prevK: 16, want: 16}, + {name: "growth past previous k", tarSize: 200 << 20, prevK: 4, want: 16}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, kForTarSize(tc.tarSize, tc.prevK)) + }) + } +} + +// Same tree gives the same ids, and the ids are pinned so a change to the fragment encoding +// is visible in review (it re-uploads every member once). +// The same tree gives the same ids, from one run to the next and from a copy in another +// directory. Ids are not pinned because owner fields are part of the fragment, so they depend +// on the user that packages the tree. +func TestV2IDsAreDeterministic(t *testing.T) { + first := v2IDs(t, writeV2Tree(t, v2TestTree(24)), v2Buckets) + assert.Equal(t, first, v2IDs(t, writeV2Tree(t, v2TestTree(24)), v2Buckets)) + assert.Equal(t, first, v2IDs(t, writeV2Tree(t, v2TestTree(24)), v2Buckets)) +} + +// fixedV2 is a v2Decision with a fixed answer. +func fixedV2(v2 bool) v2Decision { + return func(context.Context) (bool, error) { return v2, nil } +} + +// entryHeader is the part of a tar header that must match between v1 and v2. +type entryHeader struct { + Typeflag byte + Mode int64 + ModTime int64 + Uid, Gid int + Uname string + Gname string + Linkname string + Size int64 +} + +// tarHeaders reads every entry header from a tar stream. +func tarHeaders(t *testing.T, r io.Reader) map[string]entryHeader { + t.Helper() + out := map[string]entryHeader{} + tr := tar.NewReader(r) + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + return out + } + require.NoError(t, err) + out[hdr.Name] = entryHeader{ + Typeflag: hdr.Typeflag, + Mode: hdr.Mode, + ModTime: hdr.ModTime.UnixNano(), + Uid: hdr.Uid, + Gid: hdr.Gid, + Uname: hdr.Uname, + Gname: hdr.Gname, + Linkname: hdr.Linkname, + Size: hdr.Size, + } + } +} + +// A v1 tarball and the v2 members of the same tree carry identical headers for every entry: +// name, mode, mtime, owner, type, link target and size. Only the packaging differs. +func TestV2AndV1EntryHeadersMatch(t *testing.T) { + repo := writeV2Tree(t, v2TestTree(60)) + require.NoError(t, os.Symlink("pkg0/mod0.py", filepath.Join(repo, "link.py"))) + files, err := snapshotFiles(t.Context(), repo, nil, false) + require.NoError(t, err) + + v1Path := filepath.Join(t.TempDir(), "v1.tar.gz") + _, err = createPlainTarball(t.Context(), repo, v1Path, files) + require.NoError(t, err) + v1Data, err := os.ReadFile(v1Path) + require.NoError(t, err) + gz, err := gzip.NewReader(bytes.NewReader(v1Data)) + require.NoError(t, err) + v1 := tarHeaders(t, gz) + + tmp := t.TempDir() + members, err := buildV2Members(t.Context(), tmp, repo, filepath.Base(repo), files, v2Buckets) + require.NoError(t, err) + v2 := map[string]entryHeader{} + for _, m := range members { + f, err := os.Open(m.fragment) + require.NoError(t, err) + maps.Copy(v2, tarHeaders(t, f)) + require.NoError(t, f.Close()) + } + + require.Len(t, v2, len(v1)) + for name, want := range v1 { + assert.Equal(t, want, v2[name], name) + } + assert.Equal(t, byte(tar.TypeSymlink), v1[path.Join(filepath.Base(repo), "link.py")].Typeflag) +} + +// The manifest has exactly the agreed fields, and the end object is last. +func TestV2ManifestSchema(t *testing.T) { + repo := writeV2Tree(t, v2TestTree(30)) + store := newV2Store(t, nil) + files, err := snapshotFiles(t.Context(), repo, nil, false) + require.NoError(t, err) + var result snapshotResult + require.NoError(t, uploadSnapshotV2(t.Context(), store.f, testSnapshotArtifactPath+"/.internal", repo, files, &result)) + + rc, err := store.f.Read(t.Context(), path.Base(result.CodeSourcePath)) + require.NoError(t, err) + defer rc.Close() + var manifest map[string]json.RawMessage + require.NoError(t, json.NewDecoder(rc).Decode(&manifest)) + keys := make([]string, 0, len(manifest)) + for k := range manifest { + keys = append(keys, k) + } + slices.Sort(keys) + assert.Equal(t, []string{"component", "format", "k", "members", "tar_size", "version"}, keys) + + var typed struct { + Format string `json:"format"` + Version int `json:"version"` + K int `json:"k"` + Members []v2Member `json:"members"` + } + raw, err := json.Marshal(manifest) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(raw, &typed)) + assert.Equal(t, "air-snapshot-manifest", typed.Format) + assert.Equal(t, 2, typed.Version) + require.NotEmpty(t, typed.Members) + + end, err := writeV2EndFragment(t.TempDir()) + require.NoError(t, err) + assert.Equal(t, end.ID, typed.Members[len(typed.Members)-1].ID, "end object is last") + for _, m := range typed.Members { + assert.Len(t, m.ID, 32) + assert.Positive(t, m.GzSize) + assert.Positive(t, m.TarSize) + } +} + +// A one-file edit changes exactly one member id: the member holding the file. +func TestV2OneFileEditChangesOneID(t *testing.T) { + files := v2TestTree(40) + repo := writeV2Tree(t, files) + before := v2IDs(t, repo, v2Buckets) + + edited := "x = changed\n" + writeRepoFile(t, repo, "pkg1/mod5.py", edited) + require.NoError(t, os.Chmod(filepath.Join(repo, "pkg1", "mod5.py"), 0o644)) + require.NoError(t, os.Chtimes(filepath.Join(repo, "pkg1", "mod5.py"), fixedMtime, fixedMtime)) + after := v2IDs(t, repo, v2Buckets) + + added, removed := idChanges(before, after) + assert.Equal(t, 1, added, "one member id is new") + assert.Equal(t, 1, removed, "one member id is gone") + assert.Len(t, after, len(before)) +} + +// Add, remove and rename each change at most two ids. +func TestV2AddRemoveRenameChangeAtMostTwoIDs(t *testing.T) { + base := writeV2Tree(t, v2TestTree(40)) + before := v2IDs(t, base, v2Buckets) + + t.Run("add", func(t *testing.T) { + repo := writeV2Tree(t, v2TestTree(40)) + writeRepoFile(t, repo, "pkg2/new.py", "print()") + added, _ := idChanges(before, v2IDs(t, repo, v2Buckets)) + assert.LessOrEqual(t, added, 2) + }) + t.Run("remove", func(t *testing.T) { + repo := writeV2Tree(t, v2TestTree(40)) + require.NoError(t, os.Remove(filepath.Join(repo, "pkg0", "mod0.py"))) + added, _ := idChanges(before, v2IDs(t, repo, v2Buckets)) + assert.LessOrEqual(t, added, 2) + }) + t.Run("rename", func(t *testing.T) { + repo := writeV2Tree(t, v2TestTree(40)) + require.NoError(t, os.Rename(filepath.Join(repo, "pkg1", "mod1.py"), filepath.Join(repo, "pkg1", "renamed.py"))) + added, _ := idChanges(before, v2IDs(t, repo, v2Buckets)) + assert.LessOrEqual(t, added, 2) + }) +} + +// Concatenating the member objects in manifest order gives one archive with the full file +// set. Reading it with tar must not stop early, which is what record padding inside a +// fragment would cause. +func TestV2ConcatenatedMembersExtractTheFullSet(t *testing.T) { + repo := writeV2Tree(t, v2TestTree(60)) + files, err := snapshotFiles(t.Context(), repo, nil, false) + require.NoError(t, err) + tmp := t.TempDir() + members, err := buildV2Members(t.Context(), tmp, repo, filepath.Base(repo), files, v2Buckets) + require.NoError(t, err) + end, err := writeV2EndFragment(tmp) + require.NoError(t, err) + members = append(members, end) + require.Greater(t, len(members), 2) + + var concat bytes.Buffer + for _, m := range members { + gzPath := filepath.Join(tmp, m.ID+".gz") + require.NoError(t, compressRaw(t.Context(), m.fragment, gzPath)) + data, err := os.ReadFile(gzPath) + require.NoError(t, err) + concat.Write(data) + } + + var want []string + for _, f := range files { + want = append(want, path.Join(filepath.Base(repo), filepath.ToSlash(f.rel))) + } + slices.Sort(want) + + gz, err := gzip.NewReader(bytes.NewReader(concat.Bytes())) + require.NoError(t, err) + got := tarNames(t, gz) + slices.Sort(got) + assert.Equal(t, want, got) + + // The same concatenation, read by the system tar, also lists the full set. + tarBin, err := exec.LookPath("tar") + if err != nil { + t.Skip("tar not available") + } + archive := filepath.Join(tmp, "concat.tar.gz") + require.NoError(t, os.WriteFile(archive, concat.Bytes(), 0o644)) + out, err := exec.CommandContext(t.Context(), tarBin, "-tzf", archive).Output() + require.NoError(t, err) + listed := strings.Fields(string(out)) + slices.Sort(listed) + assert.Equal(t, want, listed) +} + +// The end member is two zero blocks, and it is a constant id. +func TestV2EndMemberIsConstant(t *testing.T) { + tmp := t.TempDir() + a, err := writeV2EndFragment(tmp) + require.NoError(t, err) + b, err := writeV2EndFragment(t.TempDir()) + require.NoError(t, err) + assert.Equal(t, a.ID, b.ID) + assert.Len(t, a.ID, 32) + _, err = hex.DecodeString(a.ID) + require.NoError(t, err) + assert.Equal(t, int64(v2EndTarBytes), a.TarSize) +} + +// v2Store is a fake workspace store for the v2 tests. Writes are recorded by object name, and +// the handler fails a write when failWrite says so. +type v2Store struct { + f filer.Filer + mu sync.Mutex + writes map[string]int + failWrite func(name string, attempt int) bool + failed atomic.Int32 + inFlight atomic.Int32 + maxSeen atomic.Int32 +} + +// setFailWrite changes which writes fail from now on. +func (s *v2Store) setFailWrite(failWrite func(name string, attempt int) bool) { + s.mu.Lock() + defer s.mu.Unlock() + s.failWrite = failWrite +} + +// resetWrites forgets the recorded writes. +func (s *v2Store) resetWrites() { + s.mu.Lock() + defer s.mu.Unlock() + s.writes = map[string]int{} +} + +func newV2Store(t *testing.T, failWrite func(name string, attempt int) bool) *v2Store { + t.Helper() + server := testserver.New(t) + t.Cleanup(server.Close) + s := &v2Store{writes: map[string]int{}, failWrite: failWrite} + server.Handle("POST", "/api/2.0/workspace-files/import-file/{path...}", func(req testserver.Request) any { + name := req.Vars["path"] + s.mu.Lock() + s.writes[path.Base(name)]++ + attempt := s.writes[path.Base(name)] + fail := s.failWrite + s.mu.Unlock() + if strings.Contains(name, "/.internal/") { + cur := s.inFlight.Add(1) + for { + seen := s.maxSeen.Load() + if cur <= seen || s.maxSeen.CompareAndSwap(seen, cur) { + break + } + } + defer s.inFlight.Add(-1) + time.Sleep(20 * time.Millisecond) + } + if fail != nil && fail(path.Base(name), attempt) { + s.failed.Add(1) + return testserver.Response{StatusCode: 500} + } + return req.Workspace.WorkspaceFilesImportFile(name, req.Body, req.URL.Query().Get("overwrite") == "true") + }) + testserver.AddDefaultHandlers(server) + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "token"}) + require.NoError(t, err) + s.f, err = filer.NewWorkspaceFilesClient(w, testSnapshotArtifactPath+"/.internal") + require.NoError(t, err) + return s +} + +func (s *v2Store) written(name string) int { + s.mu.Lock() + defer s.mu.Unlock() + return s.writes[name] +} + +func (s *v2Store) totalWrites() int { + s.mu.Lock() + defer s.mu.Unlock() + n := 0 + for _, c := range s.writes { + n += c + } + return n +} + +// An interrupted upload writes its objects but never the manifest, so the snapshot is not +// visible. A retry finishes it without rewriting the objects that already landed. +func TestV2InterruptedUploadLeavesNoManifest(t *testing.T) { + repo := writeV2Tree(t, v2TestTree(30)) + files, err := snapshotFiles(t.Context(), repo, nil, false) + require.NoError(t, err) + manifestName := "code_" + computePlainTarKey(files)[:16] + v2ManifestSuffix + + store := newV2Store(t, func(name string, _ int) bool { return name == manifestName }) + uploadPath := testSnapshotArtifactPath + "/.internal" + var result snapshotResult + require.Error(t, uploadSnapshotV2(t.Context(), store.f, uploadPath, repo, files, &result)) + _, err = store.f.Stat(t.Context(), manifestName) + assert.ErrorIs(t, err, os.ErrNotExist) + assert.Positive(t, store.totalWrites(), "the objects were written before the manifest failed") + + // The retry finds the objects in the store and writes only the manifest. + store.setFailWrite(nil) + store.resetWrites() + result = snapshotResult{} + require.NoError(t, uploadSnapshotV2(t.Context(), store.f, uploadPath, repo, files, &result)) + assert.Equal(t, 1, store.totalWrites(), "only the manifest is written on retry") + assert.Equal(t, 1, store.written(manifestName)) +} + +// A transient object write failure is retried and the upload succeeds. +func TestV2RetriesTransientWriteFailure(t *testing.T) { + repo := writeV2Tree(t, v2TestTree(30)) + files, err := snapshotFiles(t.Context(), repo, nil, false) + require.NoError(t, err) + failFirst := func(name string, attempt int) bool { + return strings.HasSuffix(name, ".tar.gz") && attempt == 1 + } + store := newV2Store(t, failFirst) + var result snapshotResult + require.NoError(t, uploadSnapshotV2(t.Context(), store.f, testSnapshotArtifactPath+"/.internal", repo, files, &result)) + // Every object's first write failed, so each was retried and the upload still succeeded. + assert.Positive(t, store.failed.Load()) +} + +// At most v2UploadConcurrency members are compressed or uploaded at once, and uploads do +// overlap. +func TestV2UploadConcurrencyIsBounded(t *testing.T) { + const members = 3 * v2UploadConcurrency + tmp := t.TempDir() + list := make([]v2Member, members) + for i := range list { + fragment := filepath.Join(tmp, fmt.Sprintf("f%d.tar", i)) + content := make([]byte, 1024) + content[0] = byte(i) + require.NoError(t, os.WriteFile(fragment, content, 0o644)) + sum := bytes.Repeat([]byte{byte(i + 1)}, 16) + list[i] = v2Member{ID: hex.EncodeToString(sum), TarSize: 1024, fragment: fragment} + } + store := newV2Store(t, nil) + require.NoError(t, putV2Members(t.Context(), store.f, list, tmp)) + + assert.LessOrEqual(t, int(store.maxSeen.Load()), v2UploadConcurrency) + assert.Greater(t, int(store.maxSeen.Load()), 1, "uploads should overlap") + for _, m := range list { + assert.Positive(t, m.GzSize) + } +} + +// A snapshot that is already in the store is found by its manifest name and writes nothing. +func TestV2UnchangedSnapshotWritesNothing(t *testing.T) { + repo := writeV2Tree(t, v2TestTree(30)) + store := newV2Store(t, nil) + uploadPath := testSnapshotArtifactPath + "/.internal" + files, err := snapshotFiles(t.Context(), repo, nil, false) + require.NoError(t, err) + + var first snapshotResult + require.NoError(t, uploadSnapshotV2(t.Context(), store.f, uploadPath, repo, files, &first)) + before := store.totalWrites() + assert.Positive(t, before) + + var second snapshotResult + require.NoError(t, uploadSnapshotV2(t.Context(), store.f, uploadPath, repo, files, &second)) + assert.Equal(t, first.CodeSourcePath, second.CodeSourcePath) + assert.Equal(t, before, store.totalWrites(), "unchanged resubmit writes nothing") + assert.Equal(t, new(int64(0)), second.UploadDurationMs) +} + +// The manifest records the K it was written with, and the next snapshot of the same +// component starts from it, so a shrinking repo keeps its members. +func TestV2KRatchetsFromPreviousManifest(t *testing.T) { + repo := writeV2Tree(t, v2TestTree(30)) + store := newV2Store(t, nil) + uploadPath := testSnapshotArtifactPath + "/.internal" + + // A previous snapshot of the same component with 16 members. + prev := v2Manifest{Format: v2ManifestFormat, Version: v2ManifestVersion, Component: "code", K: 16} + data, err := json.Marshal(prev) + require.NoError(t, err) + require.NoError(t, store.f.Write(t.Context(), "code_0123456789abcdef.air.json", bytes.NewReader(data), filer.OverwriteIfExists, filer.CreateParentDirectories)) + + k, err := previousV2K(t.Context(), store.f, "code") + require.NoError(t, err) + assert.Equal(t, 16, k) + + files, err := snapshotFiles(t.Context(), repo, nil, false) + require.NoError(t, err) + var result snapshotResult + require.NoError(t, uploadSnapshotV2(t.Context(), store.f, uploadPath, repo, files, &result)) + + rc, err := store.f.Read(t.Context(), path.Base(result.CodeSourcePath)) + require.NoError(t, err) + defer rc.Close() + var written v2Manifest + require.NoError(t, json.NewDecoder(rc).Decode(&written)) + assert.Equal(t, 16, written.K) + assert.Equal(t, v2ManifestFormat, written.Format) + assert.Equal(t, "code", written.Component) + // Empty buckets are omitted, and the end member is always last. + assert.Equal(t, int64(v2EndTarBytes), written.Members[len(written.Members)-1].TarSize) + assert.LessOrEqual(t, len(written.Members), 16+1) +} + +// The fetched capabilities come from the raw validate response, and absent or empty ones +// are reported as none. +func TestValidateConfigReturnsCapabilities(t *testing.T) { + tests := []struct { + name string + body any + want []string + }{ + {name: "advertised", body: map[string]any{"capabilities": []string{codeSnapshotV2Capability}}, want: []string{codeSnapshotV2Capability}}, + {name: "absent", body: map[string]any{}, want: nil}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + server := testserver.New(t) + t.Cleanup(server.Close) + server.Handle("POST", "/api/2.0/ai-training/config:validate", func(req testserver.Request) any { + return tc.body + }) + testserver.AddDefaultHandlers(server) + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "token"}) + require.NoError(t, err) + + unavailable, caps, err := preflightValidate(t.Context(), w, baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") + require.NoError(t, err) + assert.False(t, unavailable) + assert.Equal(t, tc.want, caps) + }) + } +} + +// submitWithValidate runs a plain working-tree submit against a validate stub that returns +// validateBody, and returns the bodies of the snapshot objects written and the submitted +// code_source_path. +func submitWithValidate(t *testing.T, validateBody any, repo string) (map[string][]byte, string) { + t.Helper() + server := testserver.New(t) + t.Cleanup(server.Close) + + var got jobs.SubmitRun + server.Handle("POST", "/api/2.2/jobs/runs/submit", func(req testserver.Request) any { + require.NoError(t, json.Unmarshal(req.Body, &got)) + return jobs.SubmitRunResponse{RunId: 555} + }) + server.Handle("POST", "/api/2.0/ai-training/config:validate", func(req testserver.Request) any { + return validateBody + }) + var mu sync.Mutex + bodies := map[string][]byte{} + server.Handle("POST", "/api/2.0/workspace-files/import-file/{path...}", func(req testserver.Request) any { + p := req.Vars["path"] + if strings.Contains(p, "/.air/repo_snapshots/") { + mu.Lock() + bodies[path.Base(p)] = append([]byte(nil), req.Body...) + mu.Unlock() + } + return req.Workspace.WorkspaceFilesImportFile(p, req.Body, req.URL.Query().Get("overwrite") == "true") + }) + testserver.AddDefaultHandlers(server) + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "token"}) + require.NoError(t, err) + + cfgPath := writeConfigFile(t, "run.yaml", minimalConfig+` +code_source: + type: snapshot + snapshot: + root_path: `+repo+` +`) + loaded, err := loadRunConfig(cfgPath) + require.NoError(t, err) + _, _, err = submitWorkload(cmdio.MockDiscard(t.Context()), w, loaded, cfgPath, "idem", false) + require.NoError(t, err) + return bodies, got.Tasks[0].AiRuntimeTask.CodeSourcePath +} + +// A validate response without the capability (the current backend) produces the same v1 +// tarball as before v2 existed, byte for byte, and no manifest. +func TestSubmitWithoutCapabilityKeepsV1ByteIdentical(t *testing.T) { + repo := writeV2Tree(t, v2TestTree(20)) + bodies, codeSourcePath := submitWithValidate(t, validateConfigResponse{}, repo) + + tarName := path.Base(codeSourcePath) + assert.True(t, strings.HasSuffix(tarName, ".tar.gz"), codeSourcePath) + for name := range bodies { + assert.False(t, strings.HasSuffix(name, v2ManifestSuffix), "no manifest without the capability: %s", name) + } + + files, err := snapshotFiles(t.Context(), repo, nil, false) + require.NoError(t, err) + want := filepath.Join(t.TempDir(), "want.tar.gz") + _, err = createPlainTarball(t.Context(), repo, want, files) + require.NoError(t, err) + wantBytes, err := os.ReadFile(want) + require.NoError(t, err) + assert.Equal(t, wantBytes, bodies[tarName]) +} + +// With the capability, the snapshot is a manifest plus member objects, and no v1 tarball. +func TestSubmitWithCapabilityWritesV2Manifest(t *testing.T) { + repo := writeV2Tree(t, v2TestTree(20)) + validate := map[string]any{"capabilities": []string{codeSnapshotV2Capability}} + bodies, codeSourcePath := submitWithValidate(t, validate, repo) + + assert.True(t, strings.HasSuffix(codeSourcePath, v2ManifestSuffix), codeSourcePath) + manifestName := path.Base(codeSourcePath) + raw, ok := bodies[manifestName] + require.True(t, ok, "manifest uploaded") + var manifest v2Manifest + require.NoError(t, json.Unmarshal(raw, &manifest)) + assert.Equal(t, v2ManifestFormat, manifest.Format) + assert.Equal(t, v2ManifestVersion, manifest.Version) + files, err := snapshotFiles(t.Context(), repo, nil, false) + require.NoError(t, err) + assert.NotContains(t, bodies, "code_"+computePlainTarKey(files)[:16]+".tar.gz", "no v1 tarball with the capability") +} + +// When validation is unavailable, no capability is known, so the submit keeps writing v1. +func TestSubmitWithValidateUnavailableKeepsV1(t *testing.T) { + repo := writeV2Tree(t, v2TestTree(20)) + bodies, codeSourcePath := submitWithValidate(t, testserver.Response{StatusCode: http.StatusServiceUnavailable}, repo) + + assert.True(t, strings.HasSuffix(path.Base(codeSourcePath), ".tar.gz"), codeSourcePath) + for name := range bodies { + assert.False(t, strings.HasSuffix(name, v2ManifestSuffix), "no manifest without a validated capability: %s", name) + } +} diff --git a/cmd/air/telemetry_test.go b/cmd/air/telemetry_test.go index 35a0cad984d..1b46dba57a9 100644 --- a/cmd/air/telemetry_test.go +++ b/cmd/air/telemetry_test.go @@ -180,7 +180,7 @@ func TestSnapshotPackagingFailureMeasurements(t *testing.T) { // The commit is no longer available when git archive attempts to package it. result, err := uploadSnapshotTarball(t.Context(), w, newTestRepo(t), snapshotPlan{ mode: modeGitArchive, commitSHA: "missing-commit", isGitRepo: true, - }, testSnapshotArtifactPath) + }, testSnapshotArtifactPath, fixedV2(false)) require.Error(t, err) assert.Equal(t, new(modeGitArchive), result.PackagingMode) require.NotNil(t, result.PackagingDurationMs) diff --git a/cmd/air/validateconfig.go b/cmd/air/validateconfig.go index 5033571bda2..9175168e010 100644 --- a/cmd/air/validateconfig.go +++ b/cmd/air/validateconfig.go @@ -56,6 +56,9 @@ type configFieldError struct { type validateConfigResponse struct { Errors []configFieldError `json:"errors"` + // Capabilities lists the snapshot formats the backend's launcher accepts. The SDK + // does not model the field yet, so it is read from the raw response. + Capabilities []string `json:"capabilities"` } type validationAuthenticationError struct { @@ -90,30 +93,32 @@ func validationUnavailable(err error) (*validationUnavailableError, bool) { // // Availability failures fail open: they report unavailable with a nil error, and // the caller decides how to warn. Caller failures and field errors block. -func preflightValidate(ctx context.Context, w *databricks.WorkspaceClient, cfg *runConfig, commandPath string, containers []submittedContainer, idempotencyToken string) (unavailable bool, err error) { +// +// capabilities is non-empty only when validation succeeded and the backend advertised them. +func preflightValidate(ctx context.Context, w *databricks.WorkspaceClient, cfg *runConfig, commandPath string, containers []submittedContainer, idempotencyToken string) (unavailable bool, capabilities []string, err error) { validationCtx, cancel := context.WithTimeout(ctx, submissionValidationTimeout) defer cancel() - err = validateConfigOnce(validationCtx, w, cfg, commandPath, containers, idempotencyToken) + capabilities, err = validateConfigOnce(validationCtx, w, cfg, commandPath, containers, idempotencyToken) if unavailable, _ := classifyValidationFailure(err); unavailable { - return true, nil + return true, nil, nil } - return false, err + return false, capabilities, err } -func validateConfigOnce(ctx context.Context, w *databricks.WorkspaceClient, cfg *runConfig, commandPath string, containers []submittedContainer, idempotencyToken string) error { +func validateConfigOnce(ctx context.Context, w *databricks.WorkspaceClient, cfg *runConfig, commandPath string, containers []submittedContainer, idempotencyToken string) ([]string, error) { clientCfg, err := config.HTTPClientConfigFromConfig(w.Config) if err != nil { - return fmt.Errorf("failed to create API client: %w", err) + return nil, fmt.Errorf("failed to create API client: %w", err) } requestBody, err := common.NewRequestBody(validateConfigRequest(ctx, cfg, commandPath, containers, idempotencyToken)) if err != nil { - return fmt.Errorf("failed to validate config: %w", err) + return nil, fmt.Errorf("failed to validate config: %w", err) } req, err := http.NewRequestWithContext(ctx, http.MethodPost, validateConfigPath, requestBody.Reader) if err != nil { - return fmt.Errorf("failed to validate config: %w", err) + return nil, fmt.Errorf("failed to validate config: %w", err) } req.Header.Set("Accept", "application/json") req.Header.Set("Content-Type", requestBody.ContentType) @@ -125,17 +130,17 @@ func validateConfigOnce(ctx context.Context, w *databricks.WorkspaceClient, cfg default: token, err := w.Config.GetTokenSource().Token(req.Context()) if err != nil { - return &validationAuthenticationError{fmt.Errorf("failed to validate config: %w", err)} + return nil, &validationAuthenticationError{fmt.Errorf("failed to validate config: %w", err)} } token.SetAuthHeader(req) } if err := clientCfg.AuthVisitor(req); err != nil { - return &validationAuthenticationError{fmt.Errorf("failed to validate config: %w", err)} + return nil, &validationAuthenticationError{fmt.Errorf("failed to validate config: %w", err)} } } for _, visitor := range clientCfg.Visitors { if err := visitor(req); err != nil { - return fmt.Errorf("failed to validate config: %w", err) + return nil, fmt.Errorf("failed to validate config: %w", err) } } for key, value := range auth.WorkspaceIDHeaders(w.Config) { @@ -154,35 +159,35 @@ func validateConfigOnce(ctx context.Context, w *databricks.WorkspaceClient, cfg } resp, err := (&http.Client{Transport: transport}).Do(req) if err != nil { - return fmt.Errorf("failed to validate config: %w", err) + return nil, fmt.Errorf("failed to validate config: %w", err) } defer resp.Body.Close() responseBody, err := common.NewResponseWrapper(resp, requestBody) if err != nil { err = fmt.Errorf("failed to validate config: %w", err) if errors.Is(err, context.Canceled) { - return err + return nil, err } if resp.StatusCode >= 400 { - return &apierr.APIError{ + return nil, &apierr.APIError{ StatusCode: resp.StatusCode, Message: err.Error(), } } - return asValidationUnavailable(err, true) + return nil, asValidationUnavailable(err, true) } if err := apierr.GetAPIError(ctx, responseBody); err != nil { - return fmt.Errorf("failed to validate config: %w", err) + return nil, fmt.Errorf("failed to validate config: %w", err) } var result validateConfigResponse if err := json.Unmarshal(responseBody.DebugBytes, &result); err != nil { - return asValidationUnavailable(fmt.Errorf("failed to validate config: %w", err), true) + return nil, asValidationUnavailable(fmt.Errorf("failed to validate config: %w", err), true) } if len(result.Errors) == 0 { - return nil + return result.Capabilities, nil } - return errors.New(formatConfigErrors(result.Errors)) + return nil, errors.New(formatConfigErrors(result.Errors)) } func newDryRunValidationClient(w *databricks.WorkspaceClient) (*client.DatabricksClient, error) { diff --git a/cmd/air/validateconfig_test.go b/cmd/air/validateconfig_test.go index 3e34fc3407c..f9aab81780c 100644 --- a/cmd/air/validateconfig_test.go +++ b/cmd/air/validateconfig_test.go @@ -111,7 +111,7 @@ func TestPreflightValidateFailsOpenWhenBackendUnavailable(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { srv := validateServer(t, tt.status, tt.body, nil) - _, err := preflightValidate(t.Context(), validationTestWorkspaceClient(t, srv.URL), baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") + _, _, err := preflightValidate(t.Context(), validationTestWorkspaceClient(t, srv.URL), baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") assert.NoError(t, err) }) } @@ -130,7 +130,7 @@ func TestPreflightValidationUnavailableDoesNotRetry(t *testing.T) { })) t.Cleanup(srv.Close) - unavailable, err := preflightValidate(t.Context(), validationTestWorkspaceClient(t, srv.URL), baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") + unavailable, _, err := preflightValidate(t.Context(), validationTestWorkspaceClient(t, srv.URL), baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") require.NoError(t, err) assert.True(t, unavailable) assert.Equal(t, int32(1), requests.Load()) @@ -145,7 +145,7 @@ func TestPreflightValidationDoesNotRetryTransportFailure(t *testing.T) { return nil, errors.New("connection failed") }) - _, err := preflightValidate(t.Context(), w, baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") + _, _, err := preflightValidate(t.Context(), w, baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") require.NoError(t, err) assert.Equal(t, int32(1), requests.Load()) } @@ -176,7 +176,7 @@ func TestPreflightValidationFailsOpenOnInvalidSuccessResponse(t *testing.T) { }, nil }) - _, err := preflightValidate(t.Context(), w, baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") + _, _, err := preflightValidate(t.Context(), w, baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") require.NoError(t, err) assert.Equal(t, int32(1), requests.Load()) }) @@ -201,7 +201,7 @@ func TestPreflightValidationBodyReadFailureOnCallerErrorBlocks(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond) defer cancel() - _, err := preflightValidate(ctx, w, baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") + _, _, err := preflightValidate(ctx, w, baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") require.Error(t, err) apiErr, ok := errors.AsType[*apierr.APIError](err) require.True(t, ok) @@ -237,7 +237,7 @@ func TestPreflightValidationOAuthRefreshUsesDeadline(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond) defer cancel() started := time.Now() - _, err = preflightValidate(ctx, w, baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") + _, _, err = preflightValidate(ctx, w, baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") require.NoError(t, err) assert.Less(t, time.Since(started), time.Second) @@ -283,7 +283,7 @@ func TestPreflightValidationM2MOAuthRejectionBlocks(t *testing.T) { }) require.NoError(t, err) - _, err = preflightValidate(t.Context(), w, baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") + _, _, err = preflightValidate(t.Context(), w, baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") require.ErrorIs(t, err, tt.wantErr) }) } @@ -303,7 +303,7 @@ func TestPreflightValidationTimeoutFailsOpen(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond) defer cancel() started := time.Now() - _, err := preflightValidate(ctx, validationTestWorkspaceClient(t, srv.URL), baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") + _, _, err := preflightValidate(ctx, validationTestWorkspaceClient(t, srv.URL), baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") close(releaseRequest) require.NoError(t, err) assert.Less(t, time.Since(started), time.Second) @@ -321,7 +321,7 @@ func TestPreflightValidationPropagatesCancellation(t *testing.T) { ctx, cancel := context.WithCancel(t.Context()) cancel() - _, err := preflightValidate(ctx, validationTestWorkspaceClient(t, srv.URL), baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") + _, _, err := preflightValidate(ctx, validationTestWorkspaceClient(t, srv.URL), baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") require.ErrorIs(t, err, context.Canceled) } @@ -346,7 +346,7 @@ func TestPreflightValidationPropagatesCancellationWhileReadingErrorResponse(t *t <-validationStarted cancel() }() - _, err := preflightValidate(ctx, w, baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") + _, _, err := preflightValidate(ctx, w, baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") require.ErrorIs(t, err, context.Canceled) } @@ -354,7 +354,7 @@ func TestPreflightValidateBlocksOnCallerError(t *testing.T) { for _, status := range []int{http.StatusBadRequest, http.StatusUnauthorized, http.StatusForbidden} { t.Run(http.StatusText(status), func(t *testing.T) { srv := validateServer(t, status, `{"error_code":"INVALID_PARAMETER_VALUE","message":"request rejected"}`, nil) - _, err := preflightValidate(t.Context(), validationTestWorkspaceClient(t, srv.URL), baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") + _, _, err := preflightValidate(t.Context(), validationTestWorkspaceClient(t, srv.URL), baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") require.Error(t, err) }) } @@ -367,7 +367,7 @@ func TestPreflightValidateBlocksOnRequestVisitorError(t *testing.T) { return errors.New("failed to add request headers") } - _, err := preflightValidate(t.Context(), w, baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") + _, _, err := preflightValidate(t.Context(), w, baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") require.ErrorContains(t, err, "failed to add request headers") } @@ -410,7 +410,7 @@ func TestValidateConfigRequestShape(t *testing.T) { cfg.MaxRetries = new(3) cfg.EnvVariables = map[string]string{"HF_HOME": "/tmp/hf"} cfg.Environment = &environmentConfig{UnityCatalogImage: "main.air.training:v1"} - _, err := preflightValidate(t.Context(), validationTestWorkspaceClient(t, srv.URL), cfg, "/Workspace/Users/me/cmd.sh", nil, "token") + _, _, err := preflightValidate(t.Context(), validationTestWorkspaceClient(t, srv.URL), cfg, "/Workspace/Users/me/cmd.sh", nil, "token") require.NoError(t, err) task := gotReq["task"].(map[string]any) @@ -434,7 +434,7 @@ func TestValidateConfigRequestCarriesPriorityClass(t *testing.T) { cfg := baseRunConfig() cfg.Compute.PoolID = new("cap-8xh100-res") cfg.Compute.PriorityClass = new("NORMAL") - _, err := preflightValidate(t.Context(), validationTestWorkspaceClient(t, srv.URL), cfg, "/Workspace/Users/me/cmd.sh", nil, "token") + _, _, err := preflightValidate(t.Context(), validationTestWorkspaceClient(t, srv.URL), cfg, "/Workspace/Users/me/cmd.sh", nil, "token") require.NoError(t, err) task := gotReq["task"].(map[string]any) @@ -448,7 +448,7 @@ func TestValidateConfigRequestOmitsUnsetOptions(t *testing.T) { var gotReq map[string]any srv := validateServer(t, http.StatusOK, `{}`, &gotReq) - _, err := preflightValidate(t.Context(), validationTestWorkspaceClient(t, srv.URL), baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") + _, _, err := preflightValidate(t.Context(), validationTestWorkspaceClient(t, srv.URL), baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") require.NoError(t, err) assert.Equal(t, map[string]any{"idempotency_token": "token"}, gotReq["run_options"])