From 22da4741e08faf661e8bc4b9a6093af411479a5f Mon Sep 17 00:00:00 2001 From: Steven Chen Date: Mon, 9 Nov 2026 10:08:11 +0000 Subject: [PATCH 1/2] air: write v2 code snapshots when the backend advertises them The CLI reads capabilities from the config:validate response (parsed from the raw response, since the SDK does not model the field). The snapshot stage waits for validation to finish, so the format is known before packaging. v2 is written only when validation succeeded and advertised code_snapshot_v2; otherwise v1 is written as before. v2 (plain_tar only) stores the tree as content-addressed members in 16 path-hash buckets, merged to K in {1, 4, 16} by tar size and never fewer than the previous manifest's K. Each member's id is the first 128 bits of sha256 over its uncompressed fragment, so unchanged members are not recompressed or uploaded. Members are written in parallel, with at most 8 in flight and three attempts each. The manifest is written last. Co-authored-by: Isaac --- cmd/air/runsubmit.go | 15 +- cmd/air/runsubmit_test.go | 10 +- cmd/air/snapshot_upload.go | 11 +- cmd/air/snapshot_upload_test.go | 8 +- cmd/air/snapshot_v2.go | 401 +++++++++++++++++++++++ cmd/air/snapshot_v2_test.go | 557 ++++++++++++++++++++++++++++++++ cmd/air/telemetry_test.go | 2 +- cmd/air/validateconfig.go | 43 +-- cmd/air/validateconfig_test.go | 30 +- 9 files changed, 1028 insertions(+), 49 deletions(-) create mode 100644 cmd/air/snapshot_v2.go create mode 100644 cmd/air/snapshot_v2_test.go diff --git a/cmd/air/runsubmit.go b/cmd/air/runsubmit.go index f01198d2b7a..cdab538661c 100644 --- a/cmd/air/runsubmit.go +++ b/cmd/air/runsubmit.go @@ -8,6 +8,7 @@ import ( "fmt" "net/http" "path" + "slices" "strconv" "strings" "time" @@ -359,14 +360,23 @@ func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *run // Package and upload the code snapshot, if any; the remote code_source_path // rides the ai_runtime_task. A run with no code_source leaves it empty. // Snapshot is the only code_source type. + // The snapshot format depends on the capabilities validation advertises, so the snapshot + // stage waits for validation. Validation and the other artifacts still run concurrently. + validated := make(chan struct{}) + var capabilities []string var stageSnapshot func(context.Context) (snapshotResult, error) if cfg.CodeSource != nil && cfg.CodeSource.Snapshot != nil { // Default snapshot tarballs land in the user's shared repo_snapshots dir; // uploadSnapshot replaces this when remote_volume is configured. snapshotArtifactPath := path.Join(base, ".air", "repo_snapshots") stageSnapshot = func(ctx context.Context) (snapshotResult, error) { + select { + case <-validated: + case <-ctx.Done(): + return snapshotResult{}, ctx.Err() + } // Sidecars land in the run's launch dir (funcDir) via fc, next to command.sh. - return uploadSnapshot(ctx, w, cfg.CodeSource.Snapshot, configPath, snapshotArtifactPath, fc, funcDir) + return uploadSnapshot(ctx, w, cfg.CodeSource.Snapshot, configPath, snapshotArtifactPath, fc, funcDir, slices.Contains(capabilities, codeSnapshotV2Capability)) } } @@ -377,7 +387,8 @@ func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *run group, stagingCtx := errgroup.WithContext(ctx) var validationUnavailable bool group.Go(func() (err error) { - validationUnavailable, err = preflightValidate(stagingCtx, w, cfg, commandPath, containers, token) + defer close(validated) + validationUnavailable, capabilities, err = preflightValidate(stagingCtx, w, cfg, commandPath, containers, token) return err }) group.Go(func() error { diff --git a/cmd/air/runsubmit_test.go b/cmd/air/runsubmit_test.go index e6eb46e4491..8d5ef453c98 100644 --- a/cmd/air/runsubmit_test.go +++ b/cmd/air/runsubmit_test.go @@ -862,11 +862,11 @@ code_source: ctx := cmdio.MockDiscard(t.Context()) sidecarStore, sidecarBase := testSidecarStore(t, w) - first, err := uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase) + first, err := uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase, false) require.NoError(t, err) require.NotZero(t, snapshotUploads.Load(), "first submit should upload the tarball") afterFirst := snapshotUploads.Load() - second, err := uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase) + second, err := uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase, false) require.NoError(t, err) // Content-addressed name: not the bare dir, but a 16-hex-char fingerprint. @@ -938,9 +938,9 @@ code_source: ctx := cmdio.MockDiscard(t.Context()) sidecarStore, sidecarBase := testSidecarStore(t, w) - first, err := uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase) + first, err := uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase, false) require.NoError(t, err) - second, err := uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase) + second, err := uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase, false) require.NoError(t, err) // Same pinned commit → identical content-addressed remote path, uploaded once @@ -992,7 +992,7 @@ code_source: ctx := cmdio.MockDiscard(t.Context()) sidecarStore, sidecarBase := testSidecarStore(t, w) - snap, err := uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase) + snap, err := uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase, false) require.NoError(t, err) assert.Empty(t, snap.GitStatePath) diff --git a/cmd/air/snapshot_upload.go b/cmd/air/snapshot_upload.go index fd6903e2c75..499e911c4a8 100644 --- a/cmd/air/snapshot_upload.go +++ b/cmd/air/snapshot_upload.go @@ -31,7 +31,7 @@ const uploadProvenanceSidecars = false // artifact store's .internal directory, returning the remote path to attach to the // ai_runtime_task. Uploading reuses libs/filer for the workspace/volume write without // depending on the bundle package (see uploadSnapshotTarball / snapshotUploadFiler). -func uploadSnapshot(ctx context.Context, w *databricks.WorkspaceClient, snap *snapshotSourceConfig, configPath, snapshotArtifactPath string, sidecarStore filer.Filer, sidecarBase string) (snapshotResult, error) { +func uploadSnapshot(ctx context.Context, w *databricks.WorkspaceClient, snap *snapshotSourceConfig, configPath, snapshotArtifactPath string, sidecarStore filer.Filer, sidecarBase string, codeSnapshotV2 bool) (snapshotResult, error) { repoPath, err := resolveRootPath(ctx, snap.RootPath, filepath.Dir(configPath)) if err != nil { return snapshotResult{}, err @@ -50,7 +50,7 @@ func uploadSnapshot(ctx context.Context, w *databricks.WorkspaceClient, snap *sn if snap.RemoteVolume != nil { snapshotArtifactPath = *snap.RemoteVolume } - result, err := uploadSnapshotTarball(ctx, w, repoPath, plan, snapshotArtifactPath) + result, err := uploadSnapshotTarball(ctx, w, repoPath, plan, snapshotArtifactPath, codeSnapshotV2) if err != nil { return result, err } @@ -160,7 +160,7 @@ const airArtifactInternalDir = ".internal" // The tarball name is content-addressed — by (commit, include_paths, root_path subtree) // for git_archive and by the working-tree metadata fingerprint for plain_tar — so if // the identical object already exists we skip packaging and upload entirely and reuse it. -func uploadSnapshotTarball(ctx context.Context, w *databricks.WorkspaceClient, repoPath string, plan snapshotPlan, artifactPath string) (snapshotResult, error) { +func uploadSnapshotTarball(ctx context.Context, w *databricks.WorkspaceClient, repoPath string, plan snapshotPlan, artifactPath string, codeSnapshotV2 bool) (snapshotResult, error) { result := snapshotResult{UsesGit: &plan.isGitRepo, PackagingMode: &plan.mode} f, uploadPath, err := snapshotUploadFiler(ctx, w, artifactPath) if err != nil { @@ -171,6 +171,11 @@ func uploadSnapshotTarball(ctx context.Context, w *databricks.WorkspaceClient, r if err != nil { return result, err } + // v2 is only written when the backend advertised it; any other case keeps the v1 tarball. + if codeSnapshotV2 && plan.mode == modePlainTar { + err := uploadSnapshotV2(ctx, f, uploadPath, repoPath, files, &result) + return result, err + } // code_source_path is content-addressed by tarName, so it is the same whether we // upload the bytes now or reuse an object already in the store. remote := path.Join(uploadPath, tarName) diff --git a/cmd/air/snapshot_upload_test.go b/cmd/air/snapshot_upload_test.go index 3c6c7af64f8..477444ba8db 100644 --- a/cmd/air/snapshot_upload_test.go +++ b/cmd/air/snapshot_upload_test.go @@ -85,7 +85,7 @@ func TestUploadSnapshotPlainTarWritesIndexAndTarball(t *testing.T) { ctx := cmdio.MockDiscard(t.Context()) sidecarStore, sidecarBase := testSidecarStore(t, w) - first, err := uploadSnapshot(ctx, w, snap, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase) + first, err := uploadSnapshot(ctx, w, snap, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase, false) require.NoError(t, err) tarName := path.Base(first.CodeSourcePath) @@ -97,7 +97,7 @@ func TestUploadSnapshotPlainTarWritesIndexAndTarball(t *testing.T) { assert.Len(t, rec.index.Members, 1) rec.reset() - second, err := uploadSnapshot(ctx, w, snap, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase) + second, err := uploadSnapshot(ctx, w, snap, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase, false) require.NoError(t, err) assert.Equal(t, first.CodeSourcePath, second.CodeSourcePath) assert.Empty(t, rec.names(), "cache hit should write neither the index nor the tarball") @@ -112,7 +112,7 @@ func TestUploadSnapshotPlainTarMissingSidecarReuploadsBoth(t *testing.T) { ctx := cmdio.MockDiscard(t.Context()) sidecarStore, sidecarBase := testSidecarStore(t, w) - first, err := uploadSnapshot(ctx, w, snap, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase) + first, err := uploadSnapshot(ctx, w, snap, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase, false) require.NoError(t, err) tarName := path.Base(first.CodeSourcePath) @@ -121,7 +121,7 @@ func TestUploadSnapshotPlainTarMissingSidecarReuploadsBoth(t *testing.T) { require.NoError(t, uploadDir.Delete(ctx, tarName+segmentIndexSuffix)) rec.reset() - second, err := uploadSnapshot(ctx, w, snap, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase) + second, err := uploadSnapshot(ctx, w, snap, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase, false) require.NoError(t, err) assert.Equal(t, first.CodeSourcePath, second.CodeSourcePath) assert.Equal(t, map[string]bool{tarName: true, tarName + segmentIndexSuffix: true}, rec.names()) diff --git a/cmd/air/snapshot_v2.go b/cmd/air/snapshot_v2.go new file mode 100644 index 00000000000..f1e965c614e --- /dev/null +++ b/cmd/air/snapshot_v2.go @@ -0,0 +1,401 @@ +package aircmd + +// Snapshot format v2 stores a plain_tar working tree as content-addressed members and a +// manifest, so an iteration re-uploads only the members whose files changed. The launcher +// side reads the manifest; the CLI emits v2 only when the backend advertises +// codeSnapshotV2Capability, and otherwise keeps writing v1 (see snapshot_upload.go). + +import ( + "archive/tar" + "bytes" + "context" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path" + "path/filepath" + "runtime" + "slices" + "strings" + "time" + + "github.com/databricks/cli/libs/filer" + "github.com/databricks/cli/libs/log" + "golang.org/x/sync/errgroup" +) + +// codeSnapshotV2Capability is the config:validate capability that enables v2. +const codeSnapshotV2Capability = "code_snapshot_v2" + +const ( + v2Buckets = 16 + v2ManifestFormat = "air-snapshot-manifest" + v2ManifestVersion = 2 + v2ManifestSuffix = ".air.json" + v2ObjectsDir = "objects" + v2UploadConcurrency = 8 + v2UploadAttempts = 3 + v2UploadBackoff = 200 * time.Millisecond + v2KStepBytes = 16 << 20 + v2K16StepBytes = 128 << 20 + v2EndTarBytes = 2 * tarBlockSize +) + +// v2Manifest is the object at code_source_path. Members are in archive order and end with +// the end-of-archive member, so concatenating their gzip objects gives a valid v1 .tar.gz. +type v2Manifest struct { + Format string `json:"format"` + Version int `json:"version"` + Component string `json:"component"` + K int `json:"k"` + TarSize int64 `json:"tar_size"` + Members []v2Member `json:"members"` +} + +// v2Member is one object: the first 128 bits of sha256 over its uncompressed tar fragment. +type v2Member struct { + ID string `json:"id"` + GzSize int64 `json:"gz_size"` + TarSize int64 `json:"tar_size"` + + fragment string // local uncompressed fragment, not part of the manifest +} + +// v2Bucket assigns a file to one of v2Buckets by the hash of its path, so a file keeps its +// bucket across snapshots. +func v2Bucket(rel string) int { + sum := sha256.Sum256([]byte(rel)) + return int(binary.BigEndian.Uint64(sum[:8]) % v2Buckets) +} + +// kForTarSize returns how many members a snapshot of tarSize bytes uses, never fewer than +// prevK so that a shrinking repo keeps its members. +func kForTarSize(tarSize int64, prevK int) int { + k := 1 + switch { + case tarSize >= v2K16StepBytes: + k = 16 + case tarSize >= v2KStepBytes: + k = 4 + } + return max(k, prevK) +} + +// memberOf maps a bucket to one of k members. Buckets merge evenly, so each member of a +// k=4 snapshot holds whole buckets of the k=16 layout. +func memberOf(bucket, k int) int { + return bucket / (v2Buckets / k) +} + +// v2Header is the tar header for info with owner fields cleared. The owner names come from +// the local user database and would change ids between machines. +func v2Header(info fs.FileInfo, name, linkTarget string) (*tar.Header, error) { + hdr, err := tar.FileInfoHeader(info, linkTarget) + if err != nil { + return nil, err + } + hdr.Name = name + hdr.Uid, hdr.Gid, hdr.Uname, hdr.Gname = 0, 0, "", "" + return hdr, nil +} + +// writeV2Fragment writes the tar entries for files to fragmentPath and returns the member id +// and the fragment's size. The fragment has complete entries only: no end-of-archive marker +// and no record padding, so it can be concatenated with other fragments. +func writeV2Fragment(fragmentPath, repoPath, component string, files []snapshotFile) (string, int64, error) { + f, err := os.Create(fragmentPath) + if err != nil { + return "", 0, err + } + defer f.Close() + + hash := sha256.New() + tw := tar.NewWriter(io.MultiWriter(f, hash)) + for _, file := range files { + if err := writeV2Entry(tw, repoPath, component, file); err != nil { + return "", 0, err + } + } + if err := tw.Flush(); err != nil { + return "", 0, fmt.Errorf("failed to write snapshot member: %w", err) + } + info, err := f.Stat() + if err != nil { + return "", 0, err + } + return hex.EncodeToString(hash.Sum(nil)[:16]), info.Size(), nil +} + +// writeV2Entry adds one file to the tar stream, matching tarpack's rules: regular files and +// symlinks are archived, and other file types are skipped. +func writeV2Entry(tw *tar.Writer, repoPath, component string, file snapshotFile) error { + name := filepath.Join(repoPath, file.rel) + info, err := os.Lstat(name) + if err != nil { + return fmt.Errorf("stat %s: %w", name, err) + } + mode := info.Mode() + if !mode.IsRegular() && mode&fs.ModeSymlink == 0 { + return nil + } + link := "" + if mode&fs.ModeSymlink != 0 { + if link, err = os.Readlink(name); err != nil { + return fmt.Errorf("readlink %s: %w", name, err) + } + } + hdr, err := v2Header(info, path.Join(component, filepath.ToSlash(file.rel)), link) + if err != nil { + return fmt.Errorf("tar header for %s: %w", name, err) + } + if err := tw.WriteHeader(hdr); err != nil { + return fmt.Errorf("write header for %s: %w", name, err) + } + if !mode.IsRegular() { + return nil + } + f, err := os.Open(name) + if err != nil { + return fmt.Errorf("open %s: %w", name, err) + } + defer f.Close() + if _, err := io.Copy(tw, f); err != nil { + return fmt.Errorf("write %s: %w", name, err) + } + return nil +} + +// buildV2Members groups files into k members and writes one fragment per non-empty member +// in tmp. Members are returned in index order. Empty members are omitted because a gzip +// member holding no tar entries is not a valid tar archive. +func buildV2Members(ctx context.Context, tmp, repoPath, component string, files []snapshotFile, k int) ([]v2Member, error) { + groups := make([][]snapshotFile, k) + for _, file := range files { + m := memberOf(v2Bucket(filepath.ToSlash(file.rel)), k) + groups[m] = append(groups[m], file) + } + for _, group := range groups { + slices.SortFunc(group, func(a, b snapshotFile) int { + return strings.Compare(filepath.ToSlash(a.rel), filepath.ToSlash(b.rel)) + }) + } + + members := make([]v2Member, k) + var g errgroup.Group + g.SetLimit(max(1, min(k, runtime.NumCPU()))) + for i := range k { + g.Go(func() error { + fragment := filepath.Join(tmp, fmt.Sprintf("fragment-%d.tar", i)) + id, size, err := writeV2Fragment(fragment, repoPath, component, groups[i]) + if err != nil { + return err + } + members[i] = v2Member{ID: id, TarSize: size, fragment: fragment} + return nil + }) + } + if err := g.Wait(); err != nil { + return nil, err + } + return slices.DeleteFunc(members, func(m v2Member) bool { return m.TarSize == 0 }), nil +} + +// writeV2EndFragment writes the constant end-of-archive member: two zero blocks. +func writeV2EndFragment(tmp string) (v2Member, error) { + fragment := filepath.Join(tmp, "end.tar") + if err := os.WriteFile(fragment, make([]byte, v2EndTarBytes), 0o644); err != nil { + return v2Member{}, err + } + sum := sha256.Sum256(make([]byte, v2EndTarBytes)) + return v2Member{ID: hex.EncodeToString(sum[:16]), TarSize: v2EndTarBytes, fragment: fragment}, nil +} + +// v2ObjectName is the object path of a member, relative to the snapshot directory. +func v2ObjectName(id string) string { + return path.Join(v2ObjectsDir, id+".tar.gz") +} + +// putV2Members makes every member an object in store and records its gzip size. An object +// that already exists is only sized: its id is its content, so it is never rewritten. At most +// v2UploadConcurrency members are compressed or uploaded at once. +func putV2Members(ctx context.Context, store filer.Filer, members []v2Member, tmp string) error { + g, gctx := errgroup.WithContext(ctx) + g.SetLimit(v2UploadConcurrency) + for i := range members { + g.Go(func() error { + return putV2Member(gctx, store, &members[i], tmp) + }) + } + return g.Wait() +} + +// putV2Member writes one member as an object unless it already exists. +func putV2Member(ctx context.Context, store filer.Filer, m *v2Member, tmp string) error { + name := v2ObjectName(m.ID) + info, err := store.Stat(ctx, name) + switch { + case err == nil: + m.GzSize = info.Size() + return nil + case !errors.Is(err, fs.ErrNotExist): + return fmt.Errorf("failed to check snapshot object %s: %w", name, err) + } + + gzPath := filepath.Join(tmp, m.ID+".tar.gz") + if err := compressRaw(ctx, m.fragment, gzPath); err != nil { + return fmt.Errorf("failed to compress snapshot member %s: %w", m.ID, err) + } + info, err = os.Stat(gzPath) + if err != nil { + return err + } + m.GzSize = info.Size() + return withV2Retry(ctx, func() error { + f, err := os.Open(gzPath) + if err != nil { + return err + } + defer f.Close() + return store.Write(ctx, name, f, filer.OverwriteIfExists, filer.CreateParentDirectories) + }) +} + +// withV2Retry runs op up to v2UploadAttempts times with exponential backoff. Context +// cancellation is never retried. +func withV2Retry(ctx context.Context, op func() error) error { + var err error + for attempt := range v2UploadAttempts { + if err = op(); err == nil { + return nil + } + if ctx.Err() != nil || attempt == v2UploadAttempts-1 { + break + } + log.Debugf(ctx, "snapshot upload attempt %d failed, retrying: %v", attempt+1, err) + select { + case <-time.After(v2UploadBackoff << attempt): + case <-ctx.Done(): + return ctx.Err() + } + } + return err +} + +// previousV2K returns the member count of the newest manifest for component in store, or 0 +// when there is none. K only ratchets up, so the newest manifest is enough. +func previousV2K(ctx context.Context, store filer.Filer, component string) (int, error) { + entries, err := store.ReadDir(ctx, ".") + if errors.Is(err, fs.ErrNotExist) { + return 0, nil + } + if err != nil { + return 0, fmt.Errorf("failed to list snapshots: %w", err) + } + var newest fs.DirEntry + var newestTime time.Time + for _, entry := range entries { + name := entry.Name() + if !strings.HasPrefix(name, component+"_") || !strings.HasSuffix(name, v2ManifestSuffix) { + continue + } + info, err := entry.Info() + if err != nil { + continue + } + if newest == nil || info.ModTime().After(newestTime) { + newest, newestTime = entry, info.ModTime() + } + } + if newest == nil { + return 0, nil + } + rc, err := store.Read(ctx, newest.Name()) + if err != nil { + return 0, fmt.Errorf("failed to read snapshot manifest %s: %w", newest.Name(), err) + } + defer rc.Close() + var manifest v2Manifest + if err := json.NewDecoder(rc).Decode(&manifest); err != nil { + return 0, fmt.Errorf("failed to parse snapshot manifest %s: %w", newest.Name(), err) + } + return manifest.K, nil +} + +// uploadSnapshotV2 writes the working-tree snapshot in v2 form and sets result from it. The +// manifest is written last, after every object succeeded, so a visible manifest always refers +// to complete objects. An unchanged snapshot is found by its manifest name alone. +func uploadSnapshotV2(ctx context.Context, store filer.Filer, uploadPath, repoPath string, files []snapshotFile, result *snapshotResult) error { + component := filepath.Base(repoPath) + manifestName := component + "_" + computePlainTarKey(files)[:16] + v2ManifestSuffix + remote := path.Join(uploadPath, manifestName) + + cached, err := snapshotExists(ctx, store, manifestName) + if err != nil { + return err + } + if cached { + log.Debugf(ctx, "snapshot upload skipped; reusing %s", remote) + result.CodeSourcePath = remote + result.PackagingDurationMs = new(int64(0)) + result.UploadDurationMs = new(int64(0)) + return nil + } + + prevK, err := previousV2K(ctx, store, component) + if err != nil { + return err + } + tarTotal := int64(v2EndTarBytes) + for _, file := range files { + tarTotal += file.tarSize() + } + k := kForTarSize(tarTotal, prevK) + + tmp, err := os.MkdirTemp("", "air-snapshot-v2-*") + if err != nil { + return err + } + defer os.RemoveAll(tmp) + + packagingStart := time.Now() + members, err := buildV2Members(ctx, tmp, repoPath, component, files, k) + if err != nil { + return err + } + end, err := writeV2EndFragment(tmp) + if err != nil { + return err + } + members = append(members, end) + result.PackagingDurationMs = new(time.Since(packagingStart).Milliseconds()) + + uploadStart := time.Now() + if err := putV2Members(ctx, store, members, tmp); err != nil { + return err + } + manifest := v2Manifest{Format: v2ManifestFormat, Version: v2ManifestVersion, Component: component, K: k, Members: members} + var gzTotal int64 + for _, m := range members { + manifest.TarSize += m.TarSize + gzTotal += m.GzSize + } + data, err := json.Marshal(manifest) + if err != nil { + return fmt.Errorf("failed to encode snapshot manifest: %w", err) + } + if err := withV2Retry(ctx, func() error { + return store.Write(ctx, manifestName, bytes.NewReader(data), filer.OverwriteIfExists, filer.CreateParentDirectories) + }); err != nil { + return fmt.Errorf("failed to upload snapshot manifest %s: %w", manifestName, err) + } + result.UploadDurationMs = new(time.Since(uploadStart).Milliseconds()) + result.SizeBytes = &gzTotal + result.CodeSourcePath = remote + return nil +} diff --git a/cmd/air/snapshot_v2_test.go b/cmd/air/snapshot_v2_test.go new file mode 100644 index 00000000000..06d6a89cfae --- /dev/null +++ b/cmd/air/snapshot_v2_test.go @@ -0,0 +1,557 @@ +package aircmd + +import ( + "bytes" + "compress/gzip" + "encoding/hex" + "encoding/json" + "fmt" + "net/http" + "os" + "os/exec" + "path" + "path/filepath" + "slices" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/filer" + "github.com/databricks/cli/libs/testserver" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fixedMtime pins file times so fragment bytes, and therefore ids, do not depend on when the +// test ran. +var fixedMtime = time.Unix(1_700_000_000, 0) + +// writeV2Tree writes files under a fresh repo directory with fixed modes and mtimes. The +// directory name is "code". +func writeV2Tree(t *testing.T, files map[string]string) string { + t.Helper() + repo := filepath.Join(t.TempDir(), "code") + for rel, content := range files { + writeRepoFile(t, repo, rel, content) + full := filepath.Join(repo, filepath.FromSlash(rel)) + require.NoError(t, os.Chmod(full, 0o644)) + require.NoError(t, os.Chtimes(full, fixedMtime, fixedMtime)) + } + return repo +} + +// v2TestTree is a repo with enough files to spread over several buckets. +func v2TestTree(n int) map[string]string { + files := make(map[string]string, n) + for i := range n { + files[fmt.Sprintf("pkg%d/mod%d.py", i%4, i)] = strings.Repeat(fmt.Sprintf("x = %d\n", i), 10+i) + } + return files +} + +// v2IDs returns the member ids of the repo's snapshot packed with k members, as a set. +func v2IDs(t *testing.T, repo string, k int) map[string]bool { + t.Helper() + files, err := snapshotFiles(t.Context(), repo, nil, false) + require.NoError(t, err) + members, err := buildV2Members(t.Context(), t.TempDir(), repo, filepath.Base(repo), files, k) + require.NoError(t, err) + ids := make(map[string]bool, len(members)) + for _, m := range members { + ids[m.ID] = true + } + return ids +} + +// idChanges returns how many ids in after are new, and how many in before are gone. A member +// whose content changed contributes one of each. +func idChanges(before, after map[string]bool) (added, removed int) { + for id := range after { + if !before[id] { + added++ + } + } + for id := range before { + if !after[id] { + removed++ + } + } + return added, removed +} + +func TestKForTarSize(t *testing.T) { + tests := []struct { + name string + tarSize int64 + prevK int + want int + }{ + {name: "small repo is one member", tarSize: 1 << 20, want: 1}, + {name: "16 MiB steps to four", tarSize: 16 << 20, want: 4}, + {name: "just under 128 MiB stays four", tarSize: 128<<20 - 1, want: 4}, + {name: "128 MiB steps to sixteen", tarSize: 128 << 20, want: 16}, + {name: "shrinking keeps previous k", tarSize: 1 << 20, prevK: 16, want: 16}, + {name: "growth past previous k", tarSize: 200 << 20, prevK: 4, want: 16}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, kForTarSize(tc.tarSize, tc.prevK)) + }) + } +} + +// Same tree gives the same ids, and the ids are pinned so a change to the fragment encoding +// is visible in review (it re-uploads every member once). +func TestV2IDsAreDeterministicAndPinned(t *testing.T) { + repo := writeV2Tree(t, v2TestTree(24)) + first := v2IDs(t, repo, v2Buckets) + second := v2IDs(t, repo, v2Buckets) + assert.Equal(t, first, second) + + var got []string + for id := range first { + got = append(got, id) + } + slices.Sort(got) + t.Logf("ids: %v", got) + + // Pinned for the fixed tree above (fixed mtime and mode, owner fields cleared). + want := []string{"2030798903f5ffd2d9bfd594b90f00db", "42d7e05f72052c59572de52b8685133c", "5aded820cc397b719161ef29cce01977", "7cef810a2d0a0108a199cac9b530f96c", "910cc54d61649a4f83b22932506d9a35", "91972034b47e8679e3f9c36370d8d030", "ba2ce71feaf6e8bfcc3c316df9050830", "bd4f954cb0158d38f8bc36c6902ad8a5", "d151e658c32c0de18699b99889a77982", "efec3e984af63c246333c8e1c1f96a3d", "f2c7eda8bd574178784aabddd415a0b5", "f2fb48177b6de2b085280f2ee02ee6af", "fb1516bccc909166ff6dedb7da1f50b9", "fbf370ae26a25139242b0b1840db4974"} + assert.Equal(t, want, got) +} + +// A one-file edit changes exactly one member id: the member holding the file. +func TestV2OneFileEditChangesOneID(t *testing.T) { + files := v2TestTree(40) + repo := writeV2Tree(t, files) + before := v2IDs(t, repo, v2Buckets) + + edited := "x = changed\n" + writeRepoFile(t, repo, "pkg1/mod5.py", edited) + require.NoError(t, os.Chmod(filepath.Join(repo, "pkg1", "mod5.py"), 0o644)) + require.NoError(t, os.Chtimes(filepath.Join(repo, "pkg1", "mod5.py"), fixedMtime, fixedMtime)) + after := v2IDs(t, repo, v2Buckets) + + added, removed := idChanges(before, after) + assert.Equal(t, 1, added, "one member id is new") + assert.Equal(t, 1, removed, "one member id is gone") + assert.Len(t, after, len(before)) +} + +// Add, remove and rename each change at most two ids. +func TestV2AddRemoveRenameChangeAtMostTwoIDs(t *testing.T) { + base := writeV2Tree(t, v2TestTree(40)) + before := v2IDs(t, base, v2Buckets) + + t.Run("add", func(t *testing.T) { + repo := writeV2Tree(t, v2TestTree(40)) + writeRepoFile(t, repo, "pkg2/new.py", "print()") + added, _ := idChanges(before, v2IDs(t, repo, v2Buckets)) + assert.LessOrEqual(t, added, 2) + }) + t.Run("remove", func(t *testing.T) { + repo := writeV2Tree(t, v2TestTree(40)) + require.NoError(t, os.Remove(filepath.Join(repo, "pkg0", "mod0.py"))) + added, _ := idChanges(before, v2IDs(t, repo, v2Buckets)) + assert.LessOrEqual(t, added, 2) + }) + t.Run("rename", func(t *testing.T) { + repo := writeV2Tree(t, v2TestTree(40)) + require.NoError(t, os.Rename(filepath.Join(repo, "pkg1", "mod1.py"), filepath.Join(repo, "pkg1", "renamed.py"))) + added, _ := idChanges(before, v2IDs(t, repo, v2Buckets)) + assert.LessOrEqual(t, added, 2) + }) +} + +// Concatenating the member objects in manifest order gives one archive with the full file +// set. Reading it with tar must not stop early, which is what record padding inside a +// fragment would cause. +func TestV2ConcatenatedMembersExtractTheFullSet(t *testing.T) { + repo := writeV2Tree(t, v2TestTree(60)) + files, err := snapshotFiles(t.Context(), repo, nil, false) + require.NoError(t, err) + tmp := t.TempDir() + members, err := buildV2Members(t.Context(), tmp, repo, filepath.Base(repo), files, v2Buckets) + require.NoError(t, err) + end, err := writeV2EndFragment(tmp) + require.NoError(t, err) + members = append(members, end) + require.Greater(t, len(members), 2) + + var concat bytes.Buffer + for _, m := range members { + gzPath := filepath.Join(tmp, m.ID+".gz") + require.NoError(t, compressRaw(t.Context(), m.fragment, gzPath)) + data, err := os.ReadFile(gzPath) + require.NoError(t, err) + concat.Write(data) + } + + var want []string + for _, f := range files { + want = append(want, path.Join(filepath.Base(repo), filepath.ToSlash(f.rel))) + } + slices.Sort(want) + + gz, err := gzip.NewReader(bytes.NewReader(concat.Bytes())) + require.NoError(t, err) + got := tarNames(t, gz) + slices.Sort(got) + assert.Equal(t, want, got) + + // The same concatenation, read by the system tar, also lists the full set. + tarBin, err := exec.LookPath("tar") + if err != nil { + t.Skip("tar not available") + } + archive := filepath.Join(tmp, "concat.tar.gz") + require.NoError(t, os.WriteFile(archive, concat.Bytes(), 0o644)) + out, err := exec.CommandContext(t.Context(), tarBin, "-tzf", archive).Output() + require.NoError(t, err) + listed := strings.Fields(string(out)) + slices.Sort(listed) + assert.Equal(t, want, listed) +} + +// The end member is two zero blocks, and it is a constant id. +func TestV2EndMemberIsConstant(t *testing.T) { + tmp := t.TempDir() + a, err := writeV2EndFragment(tmp) + require.NoError(t, err) + b, err := writeV2EndFragment(t.TempDir()) + require.NoError(t, err) + assert.Equal(t, a.ID, b.ID) + assert.Len(t, a.ID, 32) + _, err = hex.DecodeString(a.ID) + require.NoError(t, err) + assert.Equal(t, int64(v2EndTarBytes), a.TarSize) +} + +// v2Store is a fake workspace store for the v2 tests. Writes are recorded by object name, and +// the handler fails a write when failWrite says so. +type v2Store struct { + f filer.Filer + mu sync.Mutex + writes map[string]int + failWrite func(name string, attempt int) bool + failed atomic.Int32 + inFlight atomic.Int32 + maxSeen atomic.Int32 +} + +// setFailWrite changes which writes fail from now on. +func (s *v2Store) setFailWrite(failWrite func(name string, attempt int) bool) { + s.mu.Lock() + defer s.mu.Unlock() + s.failWrite = failWrite +} + +// resetWrites forgets the recorded writes. +func (s *v2Store) resetWrites() { + s.mu.Lock() + defer s.mu.Unlock() + s.writes = map[string]int{} +} + +func newV2Store(t *testing.T, failWrite func(name string, attempt int) bool) *v2Store { + t.Helper() + server := testserver.New(t) + t.Cleanup(server.Close) + s := &v2Store{writes: map[string]int{}, failWrite: failWrite} + server.Handle("POST", "/api/2.0/workspace-files/import-file/{path...}", func(req testserver.Request) any { + name := req.Vars["path"] + s.mu.Lock() + s.writes[path.Base(name)]++ + attempt := s.writes[path.Base(name)] + fail := s.failWrite + s.mu.Unlock() + if strings.Contains(name, "/.internal/") { + cur := s.inFlight.Add(1) + for { + seen := s.maxSeen.Load() + if cur <= seen || s.maxSeen.CompareAndSwap(seen, cur) { + break + } + } + defer s.inFlight.Add(-1) + time.Sleep(20 * time.Millisecond) + } + if fail != nil && fail(path.Base(name), attempt) { + s.failed.Add(1) + return testserver.Response{StatusCode: 500} + } + return req.Workspace.WorkspaceFilesImportFile(name, req.Body, req.URL.Query().Get("overwrite") == "true") + }) + testserver.AddDefaultHandlers(server) + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "token"}) + require.NoError(t, err) + s.f, err = filer.NewWorkspaceFilesClient(w, testSnapshotArtifactPath+"/.internal") + require.NoError(t, err) + return s +} + +func (s *v2Store) written(name string) int { + s.mu.Lock() + defer s.mu.Unlock() + return s.writes[name] +} + +func (s *v2Store) totalWrites() int { + s.mu.Lock() + defer s.mu.Unlock() + n := 0 + for _, c := range s.writes { + n += c + } + return n +} + +// An interrupted upload writes its objects but never the manifest, so the snapshot is not +// visible. A retry finishes it without rewriting the objects that already landed. +func TestV2InterruptedUploadLeavesNoManifest(t *testing.T) { + repo := writeV2Tree(t, v2TestTree(30)) + files, err := snapshotFiles(t.Context(), repo, nil, false) + require.NoError(t, err) + manifestName := "code_" + computePlainTarKey(files)[:16] + v2ManifestSuffix + + store := newV2Store(t, func(name string, _ int) bool { return name == manifestName }) + uploadPath := testSnapshotArtifactPath + "/.internal" + var result snapshotResult + require.Error(t, uploadSnapshotV2(t.Context(), store.f, uploadPath, repo, files, &result)) + _, err = store.f.Stat(t.Context(), manifestName) + assert.ErrorIs(t, err, os.ErrNotExist) + assert.Positive(t, store.totalWrites(), "the objects were written before the manifest failed") + + // The retry finds the objects in the store and writes only the manifest. + store.setFailWrite(nil) + store.resetWrites() + result = snapshotResult{} + require.NoError(t, uploadSnapshotV2(t.Context(), store.f, uploadPath, repo, files, &result)) + assert.Equal(t, 1, store.totalWrites(), "only the manifest is written on retry") + assert.Equal(t, 1, store.written(manifestName)) +} + +// A transient object write failure is retried and the upload succeeds. +func TestV2RetriesTransientWriteFailure(t *testing.T) { + repo := writeV2Tree(t, v2TestTree(30)) + files, err := snapshotFiles(t.Context(), repo, nil, false) + require.NoError(t, err) + failFirst := func(name string, attempt int) bool { + return strings.HasSuffix(name, ".tar.gz") && attempt == 1 + } + store := newV2Store(t, failFirst) + var result snapshotResult + require.NoError(t, uploadSnapshotV2(t.Context(), store.f, testSnapshotArtifactPath+"/.internal", repo, files, &result)) + // Every object's first write failed, so each was retried and the upload still succeeded. + assert.Positive(t, store.failed.Load()) +} + +// At most v2UploadConcurrency members are compressed or uploaded at once, and uploads do +// overlap. +func TestV2UploadConcurrencyIsBounded(t *testing.T) { + const members = 3 * v2UploadConcurrency + tmp := t.TempDir() + list := make([]v2Member, members) + for i := range list { + fragment := filepath.Join(tmp, fmt.Sprintf("f%d.tar", i)) + content := make([]byte, 1024) + content[0] = byte(i) + require.NoError(t, os.WriteFile(fragment, content, 0o644)) + sum := bytes.Repeat([]byte{byte(i + 1)}, 16) + list[i] = v2Member{ID: hex.EncodeToString(sum), TarSize: 1024, fragment: fragment} + } + store := newV2Store(t, nil) + require.NoError(t, putV2Members(t.Context(), store.f, list, tmp)) + + assert.LessOrEqual(t, int(store.maxSeen.Load()), v2UploadConcurrency) + assert.Greater(t, int(store.maxSeen.Load()), 1, "uploads should overlap") + for _, m := range list { + assert.Positive(t, m.GzSize) + } +} + +// A snapshot that is already in the store is found by its manifest name and writes nothing. +func TestV2UnchangedSnapshotWritesNothing(t *testing.T) { + repo := writeV2Tree(t, v2TestTree(30)) + store := newV2Store(t, nil) + uploadPath := testSnapshotArtifactPath + "/.internal" + files, err := snapshotFiles(t.Context(), repo, nil, false) + require.NoError(t, err) + + var first snapshotResult + require.NoError(t, uploadSnapshotV2(t.Context(), store.f, uploadPath, repo, files, &first)) + before := store.totalWrites() + assert.Positive(t, before) + + var second snapshotResult + require.NoError(t, uploadSnapshotV2(t.Context(), store.f, uploadPath, repo, files, &second)) + assert.Equal(t, first.CodeSourcePath, second.CodeSourcePath) + assert.Equal(t, before, store.totalWrites(), "unchanged resubmit writes nothing") + assert.Equal(t, new(int64(0)), second.UploadDurationMs) +} + +// The manifest records the K it was written with, and the next snapshot of the same +// component starts from it, so a shrinking repo keeps its members. +func TestV2KRatchetsFromPreviousManifest(t *testing.T) { + repo := writeV2Tree(t, v2TestTree(30)) + store := newV2Store(t, nil) + uploadPath := testSnapshotArtifactPath + "/.internal" + + // A previous snapshot of the same component with 16 members. + prev := v2Manifest{Format: v2ManifestFormat, Version: v2ManifestVersion, Component: "code", K: 16} + data, err := json.Marshal(prev) + require.NoError(t, err) + require.NoError(t, store.f.Write(t.Context(), "code_0123456789abcdef.air.json", bytes.NewReader(data), filer.OverwriteIfExists, filer.CreateParentDirectories)) + + k, err := previousV2K(t.Context(), store.f, "code") + require.NoError(t, err) + assert.Equal(t, 16, k) + + files, err := snapshotFiles(t.Context(), repo, nil, false) + require.NoError(t, err) + var result snapshotResult + require.NoError(t, uploadSnapshotV2(t.Context(), store.f, uploadPath, repo, files, &result)) + + rc, err := store.f.Read(t.Context(), path.Base(result.CodeSourcePath)) + require.NoError(t, err) + defer rc.Close() + var written v2Manifest + require.NoError(t, json.NewDecoder(rc).Decode(&written)) + assert.Equal(t, 16, written.K) + assert.Equal(t, v2ManifestFormat, written.Format) + assert.Equal(t, "code", written.Component) + // Empty buckets are omitted, and the end member is always last. + assert.Equal(t, int64(v2EndTarBytes), written.Members[len(written.Members)-1].TarSize) + assert.LessOrEqual(t, len(written.Members), 16+1) +} + +// The fetched capabilities come from the raw validate response, and absent or empty ones +// are reported as none. +func TestValidateConfigReturnsCapabilities(t *testing.T) { + tests := []struct { + name string + body any + want []string + }{ + {name: "advertised", body: map[string]any{"capabilities": []string{codeSnapshotV2Capability}}, want: []string{codeSnapshotV2Capability}}, + {name: "absent", body: map[string]any{}, want: nil}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + server := testserver.New(t) + t.Cleanup(server.Close) + server.Handle("POST", "/api/2.0/ai-training/config:validate", func(req testserver.Request) any { + return tc.body + }) + testserver.AddDefaultHandlers(server) + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "token"}) + require.NoError(t, err) + + unavailable, caps, err := preflightValidate(t.Context(), w, baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") + require.NoError(t, err) + assert.False(t, unavailable) + assert.Equal(t, tc.want, caps) + }) + } +} + +// submitWithValidate runs a plain working-tree submit against a validate stub that returns +// validateBody, and returns the bodies of the snapshot objects written and the submitted +// code_source_path. +func submitWithValidate(t *testing.T, validateBody any, repo string) (map[string][]byte, string) { + t.Helper() + server := testserver.New(t) + t.Cleanup(server.Close) + + var got jobs.SubmitRun + server.Handle("POST", "/api/2.2/jobs/runs/submit", func(req testserver.Request) any { + require.NoError(t, json.Unmarshal(req.Body, &got)) + return jobs.SubmitRunResponse{RunId: 555} + }) + server.Handle("POST", "/api/2.0/ai-training/config:validate", func(req testserver.Request) any { + return validateBody + }) + var mu sync.Mutex + bodies := map[string][]byte{} + server.Handle("POST", "/api/2.0/workspace-files/import-file/{path...}", func(req testserver.Request) any { + p := req.Vars["path"] + if strings.Contains(p, "/.air/repo_snapshots/") { + mu.Lock() + bodies[path.Base(p)] = append([]byte(nil), req.Body...) + mu.Unlock() + } + return req.Workspace.WorkspaceFilesImportFile(p, req.Body, req.URL.Query().Get("overwrite") == "true") + }) + testserver.AddDefaultHandlers(server) + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "token"}) + require.NoError(t, err) + + cfgPath := writeConfigFile(t, "run.yaml", minimalConfig+` +code_source: + type: snapshot + snapshot: + root_path: `+repo+` +`) + loaded, err := loadRunConfig(cfgPath) + require.NoError(t, err) + _, _, err = submitWorkload(cmdio.MockDiscard(t.Context()), w, loaded, cfgPath, "idem", false) + require.NoError(t, err) + return bodies, got.Tasks[0].AiRuntimeTask.CodeSourcePath +} + +// A validate response without the capability (the current backend) produces the same v1 +// tarball as before v2 existed, byte for byte, and no manifest. +func TestSubmitWithoutCapabilityKeepsV1ByteIdentical(t *testing.T) { + repo := writeV2Tree(t, v2TestTree(20)) + bodies, codeSourcePath := submitWithValidate(t, validateConfigResponse{}, repo) + + tarName := path.Base(codeSourcePath) + assert.True(t, strings.HasSuffix(tarName, ".tar.gz"), codeSourcePath) + for name := range bodies { + assert.False(t, strings.HasSuffix(name, v2ManifestSuffix), "no manifest without the capability: %s", name) + } + + files, err := snapshotFiles(t.Context(), repo, nil, false) + require.NoError(t, err) + want := filepath.Join(t.TempDir(), "want.tar.gz") + _, err = createPlainTarball(t.Context(), repo, want, files) + require.NoError(t, err) + wantBytes, err := os.ReadFile(want) + require.NoError(t, err) + assert.Equal(t, wantBytes, bodies[tarName]) +} + +// With the capability, the snapshot is a manifest plus member objects, and no v1 tarball. +func TestSubmitWithCapabilityWritesV2Manifest(t *testing.T) { + repo := writeV2Tree(t, v2TestTree(20)) + validate := map[string]any{"capabilities": []string{codeSnapshotV2Capability}} + bodies, codeSourcePath := submitWithValidate(t, validate, repo) + + assert.True(t, strings.HasSuffix(codeSourcePath, v2ManifestSuffix), codeSourcePath) + manifestName := path.Base(codeSourcePath) + raw, ok := bodies[manifestName] + require.True(t, ok, "manifest uploaded") + var manifest v2Manifest + require.NoError(t, json.Unmarshal(raw, &manifest)) + assert.Equal(t, v2ManifestFormat, manifest.Format) + assert.Equal(t, v2ManifestVersion, manifest.Version) + files, err := snapshotFiles(t.Context(), repo, nil, false) + require.NoError(t, err) + assert.NotContains(t, bodies, "code_"+computePlainTarKey(files)[:16]+".tar.gz", "no v1 tarball with the capability") +} + +// When validation is unavailable, no capability is known, so the submit keeps writing v1. +func TestSubmitWithValidateUnavailableKeepsV1(t *testing.T) { + repo := writeV2Tree(t, v2TestTree(20)) + bodies, codeSourcePath := submitWithValidate(t, testserver.Response{StatusCode: http.StatusServiceUnavailable}, repo) + + assert.True(t, strings.HasSuffix(path.Base(codeSourcePath), ".tar.gz"), codeSourcePath) + for name := range bodies { + assert.False(t, strings.HasSuffix(name, v2ManifestSuffix), "no manifest without a validated capability: %s", name) + } +} diff --git a/cmd/air/telemetry_test.go b/cmd/air/telemetry_test.go index 35a0cad984d..081af188483 100644 --- a/cmd/air/telemetry_test.go +++ b/cmd/air/telemetry_test.go @@ -180,7 +180,7 @@ func TestSnapshotPackagingFailureMeasurements(t *testing.T) { // The commit is no longer available when git archive attempts to package it. result, err := uploadSnapshotTarball(t.Context(), w, newTestRepo(t), snapshotPlan{ mode: modeGitArchive, commitSHA: "missing-commit", isGitRepo: true, - }, testSnapshotArtifactPath) + }, testSnapshotArtifactPath, false) require.Error(t, err) assert.Equal(t, new(modeGitArchive), result.PackagingMode) require.NotNil(t, result.PackagingDurationMs) diff --git a/cmd/air/validateconfig.go b/cmd/air/validateconfig.go index 5033571bda2..9175168e010 100644 --- a/cmd/air/validateconfig.go +++ b/cmd/air/validateconfig.go @@ -56,6 +56,9 @@ type configFieldError struct { type validateConfigResponse struct { Errors []configFieldError `json:"errors"` + // Capabilities lists the snapshot formats the backend's launcher accepts. The SDK + // does not model the field yet, so it is read from the raw response. + Capabilities []string `json:"capabilities"` } type validationAuthenticationError struct { @@ -90,30 +93,32 @@ func validationUnavailable(err error) (*validationUnavailableError, bool) { // // Availability failures fail open: they report unavailable with a nil error, and // the caller decides how to warn. Caller failures and field errors block. -func preflightValidate(ctx context.Context, w *databricks.WorkspaceClient, cfg *runConfig, commandPath string, containers []submittedContainer, idempotencyToken string) (unavailable bool, err error) { +// +// capabilities is non-empty only when validation succeeded and the backend advertised them. +func preflightValidate(ctx context.Context, w *databricks.WorkspaceClient, cfg *runConfig, commandPath string, containers []submittedContainer, idempotencyToken string) (unavailable bool, capabilities []string, err error) { validationCtx, cancel := context.WithTimeout(ctx, submissionValidationTimeout) defer cancel() - err = validateConfigOnce(validationCtx, w, cfg, commandPath, containers, idempotencyToken) + capabilities, err = validateConfigOnce(validationCtx, w, cfg, commandPath, containers, idempotencyToken) if unavailable, _ := classifyValidationFailure(err); unavailable { - return true, nil + return true, nil, nil } - return false, err + return false, capabilities, err } -func validateConfigOnce(ctx context.Context, w *databricks.WorkspaceClient, cfg *runConfig, commandPath string, containers []submittedContainer, idempotencyToken string) error { +func validateConfigOnce(ctx context.Context, w *databricks.WorkspaceClient, cfg *runConfig, commandPath string, containers []submittedContainer, idempotencyToken string) ([]string, error) { clientCfg, err := config.HTTPClientConfigFromConfig(w.Config) if err != nil { - return fmt.Errorf("failed to create API client: %w", err) + return nil, fmt.Errorf("failed to create API client: %w", err) } requestBody, err := common.NewRequestBody(validateConfigRequest(ctx, cfg, commandPath, containers, idempotencyToken)) if err != nil { - return fmt.Errorf("failed to validate config: %w", err) + return nil, fmt.Errorf("failed to validate config: %w", err) } req, err := http.NewRequestWithContext(ctx, http.MethodPost, validateConfigPath, requestBody.Reader) if err != nil { - return fmt.Errorf("failed to validate config: %w", err) + return nil, fmt.Errorf("failed to validate config: %w", err) } req.Header.Set("Accept", "application/json") req.Header.Set("Content-Type", requestBody.ContentType) @@ -125,17 +130,17 @@ func validateConfigOnce(ctx context.Context, w *databricks.WorkspaceClient, cfg default: token, err := w.Config.GetTokenSource().Token(req.Context()) if err != nil { - return &validationAuthenticationError{fmt.Errorf("failed to validate config: %w", err)} + return nil, &validationAuthenticationError{fmt.Errorf("failed to validate config: %w", err)} } token.SetAuthHeader(req) } if err := clientCfg.AuthVisitor(req); err != nil { - return &validationAuthenticationError{fmt.Errorf("failed to validate config: %w", err)} + return nil, &validationAuthenticationError{fmt.Errorf("failed to validate config: %w", err)} } } for _, visitor := range clientCfg.Visitors { if err := visitor(req); err != nil { - return fmt.Errorf("failed to validate config: %w", err) + return nil, fmt.Errorf("failed to validate config: %w", err) } } for key, value := range auth.WorkspaceIDHeaders(w.Config) { @@ -154,35 +159,35 @@ func validateConfigOnce(ctx context.Context, w *databricks.WorkspaceClient, cfg } resp, err := (&http.Client{Transport: transport}).Do(req) if err != nil { - return fmt.Errorf("failed to validate config: %w", err) + return nil, fmt.Errorf("failed to validate config: %w", err) } defer resp.Body.Close() responseBody, err := common.NewResponseWrapper(resp, requestBody) if err != nil { err = fmt.Errorf("failed to validate config: %w", err) if errors.Is(err, context.Canceled) { - return err + return nil, err } if resp.StatusCode >= 400 { - return &apierr.APIError{ + return nil, &apierr.APIError{ StatusCode: resp.StatusCode, Message: err.Error(), } } - return asValidationUnavailable(err, true) + return nil, asValidationUnavailable(err, true) } if err := apierr.GetAPIError(ctx, responseBody); err != nil { - return fmt.Errorf("failed to validate config: %w", err) + return nil, fmt.Errorf("failed to validate config: %w", err) } var result validateConfigResponse if err := json.Unmarshal(responseBody.DebugBytes, &result); err != nil { - return asValidationUnavailable(fmt.Errorf("failed to validate config: %w", err), true) + return nil, asValidationUnavailable(fmt.Errorf("failed to validate config: %w", err), true) } if len(result.Errors) == 0 { - return nil + return result.Capabilities, nil } - return errors.New(formatConfigErrors(result.Errors)) + return nil, errors.New(formatConfigErrors(result.Errors)) } func newDryRunValidationClient(w *databricks.WorkspaceClient) (*client.DatabricksClient, error) { diff --git a/cmd/air/validateconfig_test.go b/cmd/air/validateconfig_test.go index 3e34fc3407c..f9aab81780c 100644 --- a/cmd/air/validateconfig_test.go +++ b/cmd/air/validateconfig_test.go @@ -111,7 +111,7 @@ func TestPreflightValidateFailsOpenWhenBackendUnavailable(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { srv := validateServer(t, tt.status, tt.body, nil) - _, err := preflightValidate(t.Context(), validationTestWorkspaceClient(t, srv.URL), baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") + _, _, err := preflightValidate(t.Context(), validationTestWorkspaceClient(t, srv.URL), baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") assert.NoError(t, err) }) } @@ -130,7 +130,7 @@ func TestPreflightValidationUnavailableDoesNotRetry(t *testing.T) { })) t.Cleanup(srv.Close) - unavailable, err := preflightValidate(t.Context(), validationTestWorkspaceClient(t, srv.URL), baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") + unavailable, _, err := preflightValidate(t.Context(), validationTestWorkspaceClient(t, srv.URL), baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") require.NoError(t, err) assert.True(t, unavailable) assert.Equal(t, int32(1), requests.Load()) @@ -145,7 +145,7 @@ func TestPreflightValidationDoesNotRetryTransportFailure(t *testing.T) { return nil, errors.New("connection failed") }) - _, err := preflightValidate(t.Context(), w, baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") + _, _, err := preflightValidate(t.Context(), w, baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") require.NoError(t, err) assert.Equal(t, int32(1), requests.Load()) } @@ -176,7 +176,7 @@ func TestPreflightValidationFailsOpenOnInvalidSuccessResponse(t *testing.T) { }, nil }) - _, err := preflightValidate(t.Context(), w, baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") + _, _, err := preflightValidate(t.Context(), w, baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") require.NoError(t, err) assert.Equal(t, int32(1), requests.Load()) }) @@ -201,7 +201,7 @@ func TestPreflightValidationBodyReadFailureOnCallerErrorBlocks(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond) defer cancel() - _, err := preflightValidate(ctx, w, baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") + _, _, err := preflightValidate(ctx, w, baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") require.Error(t, err) apiErr, ok := errors.AsType[*apierr.APIError](err) require.True(t, ok) @@ -237,7 +237,7 @@ func TestPreflightValidationOAuthRefreshUsesDeadline(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond) defer cancel() started := time.Now() - _, err = preflightValidate(ctx, w, baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") + _, _, err = preflightValidate(ctx, w, baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") require.NoError(t, err) assert.Less(t, time.Since(started), time.Second) @@ -283,7 +283,7 @@ func TestPreflightValidationM2MOAuthRejectionBlocks(t *testing.T) { }) require.NoError(t, err) - _, err = preflightValidate(t.Context(), w, baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") + _, _, err = preflightValidate(t.Context(), w, baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") require.ErrorIs(t, err, tt.wantErr) }) } @@ -303,7 +303,7 @@ func TestPreflightValidationTimeoutFailsOpen(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond) defer cancel() started := time.Now() - _, err := preflightValidate(ctx, validationTestWorkspaceClient(t, srv.URL), baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") + _, _, err := preflightValidate(ctx, validationTestWorkspaceClient(t, srv.URL), baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") close(releaseRequest) require.NoError(t, err) assert.Less(t, time.Since(started), time.Second) @@ -321,7 +321,7 @@ func TestPreflightValidationPropagatesCancellation(t *testing.T) { ctx, cancel := context.WithCancel(t.Context()) cancel() - _, err := preflightValidate(ctx, validationTestWorkspaceClient(t, srv.URL), baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") + _, _, err := preflightValidate(ctx, validationTestWorkspaceClient(t, srv.URL), baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") require.ErrorIs(t, err, context.Canceled) } @@ -346,7 +346,7 @@ func TestPreflightValidationPropagatesCancellationWhileReadingErrorResponse(t *t <-validationStarted cancel() }() - _, err := preflightValidate(ctx, w, baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") + _, _, err := preflightValidate(ctx, w, baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") require.ErrorIs(t, err, context.Canceled) } @@ -354,7 +354,7 @@ func TestPreflightValidateBlocksOnCallerError(t *testing.T) { for _, status := range []int{http.StatusBadRequest, http.StatusUnauthorized, http.StatusForbidden} { t.Run(http.StatusText(status), func(t *testing.T) { srv := validateServer(t, status, `{"error_code":"INVALID_PARAMETER_VALUE","message":"request rejected"}`, nil) - _, err := preflightValidate(t.Context(), validationTestWorkspaceClient(t, srv.URL), baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") + _, _, err := preflightValidate(t.Context(), validationTestWorkspaceClient(t, srv.URL), baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") require.Error(t, err) }) } @@ -367,7 +367,7 @@ func TestPreflightValidateBlocksOnRequestVisitorError(t *testing.T) { return errors.New("failed to add request headers") } - _, err := preflightValidate(t.Context(), w, baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") + _, _, err := preflightValidate(t.Context(), w, baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") require.ErrorContains(t, err, "failed to add request headers") } @@ -410,7 +410,7 @@ func TestValidateConfigRequestShape(t *testing.T) { cfg.MaxRetries = new(3) cfg.EnvVariables = map[string]string{"HF_HOME": "/tmp/hf"} cfg.Environment = &environmentConfig{UnityCatalogImage: "main.air.training:v1"} - _, err := preflightValidate(t.Context(), validationTestWorkspaceClient(t, srv.URL), cfg, "/Workspace/Users/me/cmd.sh", nil, "token") + _, _, err := preflightValidate(t.Context(), validationTestWorkspaceClient(t, srv.URL), cfg, "/Workspace/Users/me/cmd.sh", nil, "token") require.NoError(t, err) task := gotReq["task"].(map[string]any) @@ -434,7 +434,7 @@ func TestValidateConfigRequestCarriesPriorityClass(t *testing.T) { cfg := baseRunConfig() cfg.Compute.PoolID = new("cap-8xh100-res") cfg.Compute.PriorityClass = new("NORMAL") - _, err := preflightValidate(t.Context(), validationTestWorkspaceClient(t, srv.URL), cfg, "/Workspace/Users/me/cmd.sh", nil, "token") + _, _, err := preflightValidate(t.Context(), validationTestWorkspaceClient(t, srv.URL), cfg, "/Workspace/Users/me/cmd.sh", nil, "token") require.NoError(t, err) task := gotReq["task"].(map[string]any) @@ -448,7 +448,7 @@ func TestValidateConfigRequestOmitsUnsetOptions(t *testing.T) { var gotReq map[string]any srv := validateServer(t, http.StatusOK, `{}`, &gotReq) - _, err := preflightValidate(t.Context(), validationTestWorkspaceClient(t, srv.URL), baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") + _, _, err := preflightValidate(t.Context(), validationTestWorkspaceClient(t, srv.URL), baseRunConfig(), "/Workspace/Users/me/cmd.sh", nil, "token") require.NoError(t, err) assert.Equal(t, map[string]any{"idempotency_token": "token"}, gotReq["run_options"]) From bc8c8ed1bd2b04e101f2696fc2ae11ec0f461f52 Mon Sep 17 00:00:00 2001 From: Steven Chen Date: Mon, 9 Nov 2026 10:15:48 +0000 Subject: [PATCH 2/2] air: wait for the capability only at the format decision; keep v1 owner fields in v2 The snapshot stage no longer waits for config:validate up front. Planning, listing and keying run in parallel with validation, and the wait happens at the v1/v2 decision, before the cache check. git_archive never waits. v2 fragments carry the same owner fields as v1 (uid, gid, uname, gname), so extraction restores the same ownership. Ids therefore depend on the packaging user and machine, and another machine re-uploads once. Co-authored-by: Isaac --- cmd/air/runsubmit.go | 14 +-- cmd/air/runsubmit_test.go | 10 +-- cmd/air/snapshot_upload.go | 21 +++-- cmd/air/snapshot_upload_test.go | 8 +- cmd/air/snapshot_v2.go | 10 ++- cmd/air/snapshot_v2_test.go | 145 +++++++++++++++++++++++++++++--- cmd/air/telemetry_test.go | 2 +- 7 files changed, 171 insertions(+), 39 deletions(-) diff --git a/cmd/air/runsubmit.go b/cmd/air/runsubmit.go index cdab538661c..451772da098 100644 --- a/cmd/air/runsubmit.go +++ b/cmd/air/runsubmit.go @@ -360,8 +360,9 @@ func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *run // Package and upload the code snapshot, if any; the remote code_source_path // rides the ai_runtime_task. A run with no code_source leaves it empty. // Snapshot is the only code_source type. - // The snapshot format depends on the capabilities validation advertises, so the snapshot - // stage waits for validation. Validation and the other artifacts still run concurrently. + // The snapshot format depends on the capabilities validation advertises. Only the format + // decision waits for validation: planning, listing and keying the snapshot run in parallel + // with it, and v1 submits do not wait longer than before. validated := make(chan struct{}) var capabilities []string var stageSnapshot func(context.Context) (snapshotResult, error) @@ -369,14 +370,17 @@ func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *run // Default snapshot tarballs land in the user's shared repo_snapshots dir; // uploadSnapshot replaces this when remote_volume is configured. snapshotArtifactPath := path.Join(base, ".air", "repo_snapshots") - stageSnapshot = func(ctx context.Context) (snapshotResult, error) { + v2Advertised := func(ctx context.Context) (bool, error) { select { case <-validated: case <-ctx.Done(): - return snapshotResult{}, ctx.Err() + return false, ctx.Err() } + return slices.Contains(capabilities, codeSnapshotV2Capability), nil + } + stageSnapshot = func(ctx context.Context) (snapshotResult, error) { // Sidecars land in the run's launch dir (funcDir) via fc, next to command.sh. - return uploadSnapshot(ctx, w, cfg.CodeSource.Snapshot, configPath, snapshotArtifactPath, fc, funcDir, slices.Contains(capabilities, codeSnapshotV2Capability)) + return uploadSnapshot(ctx, w, cfg.CodeSource.Snapshot, configPath, snapshotArtifactPath, fc, funcDir, v2Advertised) } } diff --git a/cmd/air/runsubmit_test.go b/cmd/air/runsubmit_test.go index 8d5ef453c98..87116fde10e 100644 --- a/cmd/air/runsubmit_test.go +++ b/cmd/air/runsubmit_test.go @@ -862,11 +862,11 @@ code_source: ctx := cmdio.MockDiscard(t.Context()) sidecarStore, sidecarBase := testSidecarStore(t, w) - first, err := uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase, false) + first, err := uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase, fixedV2(false)) require.NoError(t, err) require.NotZero(t, snapshotUploads.Load(), "first submit should upload the tarball") afterFirst := snapshotUploads.Load() - second, err := uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase, false) + second, err := uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase, fixedV2(false)) require.NoError(t, err) // Content-addressed name: not the bare dir, but a 16-hex-char fingerprint. @@ -938,9 +938,9 @@ code_source: ctx := cmdio.MockDiscard(t.Context()) sidecarStore, sidecarBase := testSidecarStore(t, w) - first, err := uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase, false) + first, err := uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase, fixedV2(false)) require.NoError(t, err) - second, err := uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase, false) + second, err := uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase, fixedV2(false)) require.NoError(t, err) // Same pinned commit → identical content-addressed remote path, uploaded once @@ -992,7 +992,7 @@ code_source: ctx := cmdio.MockDiscard(t.Context()) sidecarStore, sidecarBase := testSidecarStore(t, w) - snap, err := uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase, false) + snap, err := uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase, fixedV2(false)) require.NoError(t, err) assert.Empty(t, snap.GitStatePath) diff --git a/cmd/air/snapshot_upload.go b/cmd/air/snapshot_upload.go index 499e911c4a8..c21b5b7c0c3 100644 --- a/cmd/air/snapshot_upload.go +++ b/cmd/air/snapshot_upload.go @@ -31,7 +31,7 @@ const uploadProvenanceSidecars = false // artifact store's .internal directory, returning the remote path to attach to the // ai_runtime_task. Uploading reuses libs/filer for the workspace/volume write without // depending on the bundle package (see uploadSnapshotTarball / snapshotUploadFiler). -func uploadSnapshot(ctx context.Context, w *databricks.WorkspaceClient, snap *snapshotSourceConfig, configPath, snapshotArtifactPath string, sidecarStore filer.Filer, sidecarBase string, codeSnapshotV2 bool) (snapshotResult, error) { +func uploadSnapshot(ctx context.Context, w *databricks.WorkspaceClient, snap *snapshotSourceConfig, configPath, snapshotArtifactPath string, sidecarStore filer.Filer, sidecarBase string, v2Advertised v2Decision) (snapshotResult, error) { repoPath, err := resolveRootPath(ctx, snap.RootPath, filepath.Dir(configPath)) if err != nil { return snapshotResult{}, err @@ -50,7 +50,7 @@ func uploadSnapshot(ctx context.Context, w *databricks.WorkspaceClient, snap *sn if snap.RemoteVolume != nil { snapshotArtifactPath = *snap.RemoteVolume } - result, err := uploadSnapshotTarball(ctx, w, repoPath, plan, snapshotArtifactPath, codeSnapshotV2) + result, err := uploadSnapshotTarball(ctx, w, repoPath, plan, snapshotArtifactPath, v2Advertised) if err != nil { return result, err } @@ -160,7 +160,7 @@ const airArtifactInternalDir = ".internal" // The tarball name is content-addressed — by (commit, include_paths, root_path subtree) // for git_archive and by the working-tree metadata fingerprint for plain_tar — so if // the identical object already exists we skip packaging and upload entirely and reuse it. -func uploadSnapshotTarball(ctx context.Context, w *databricks.WorkspaceClient, repoPath string, plan snapshotPlan, artifactPath string, codeSnapshotV2 bool) (snapshotResult, error) { +func uploadSnapshotTarball(ctx context.Context, w *databricks.WorkspaceClient, repoPath string, plan snapshotPlan, artifactPath string, v2Advertised v2Decision) (snapshotResult, error) { result := snapshotResult{UsesGit: &plan.isGitRepo, PackagingMode: &plan.mode} f, uploadPath, err := snapshotUploadFiler(ctx, w, artifactPath) if err != nil { @@ -171,10 +171,17 @@ func uploadSnapshotTarball(ctx context.Context, w *databricks.WorkspaceClient, r if err != nil { return result, err } - // v2 is only written when the backend advertised it; any other case keeps the v1 tarball. - if codeSnapshotV2 && plan.mode == modePlainTar { - err := uploadSnapshotV2(ctx, f, uploadPath, repoPath, files, &result) - return result, err + // The format decision is the first point that needs validation. Only plain_tar can be v2, so + // git_archive never waits. The cache check below differs by format, so it comes after. + if plan.mode == modePlainTar { + v2, err := v2Advertised(ctx) + if err != nil { + return result, err + } + if v2 { + err := uploadSnapshotV2(ctx, f, uploadPath, repoPath, files, &result) + return result, err + } } // code_source_path is content-addressed by tarName, so it is the same whether we // upload the bytes now or reuse an object already in the store. diff --git a/cmd/air/snapshot_upload_test.go b/cmd/air/snapshot_upload_test.go index 477444ba8db..4f9eb6695df 100644 --- a/cmd/air/snapshot_upload_test.go +++ b/cmd/air/snapshot_upload_test.go @@ -85,7 +85,7 @@ func TestUploadSnapshotPlainTarWritesIndexAndTarball(t *testing.T) { ctx := cmdio.MockDiscard(t.Context()) sidecarStore, sidecarBase := testSidecarStore(t, w) - first, err := uploadSnapshot(ctx, w, snap, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase, false) + first, err := uploadSnapshot(ctx, w, snap, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase, fixedV2(false)) require.NoError(t, err) tarName := path.Base(first.CodeSourcePath) @@ -97,7 +97,7 @@ func TestUploadSnapshotPlainTarWritesIndexAndTarball(t *testing.T) { assert.Len(t, rec.index.Members, 1) rec.reset() - second, err := uploadSnapshot(ctx, w, snap, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase, false) + second, err := uploadSnapshot(ctx, w, snap, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase, fixedV2(false)) require.NoError(t, err) assert.Equal(t, first.CodeSourcePath, second.CodeSourcePath) assert.Empty(t, rec.names(), "cache hit should write neither the index nor the tarball") @@ -112,7 +112,7 @@ func TestUploadSnapshotPlainTarMissingSidecarReuploadsBoth(t *testing.T) { ctx := cmdio.MockDiscard(t.Context()) sidecarStore, sidecarBase := testSidecarStore(t, w) - first, err := uploadSnapshot(ctx, w, snap, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase, false) + first, err := uploadSnapshot(ctx, w, snap, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase, fixedV2(false)) require.NoError(t, err) tarName := path.Base(first.CodeSourcePath) @@ -121,7 +121,7 @@ func TestUploadSnapshotPlainTarMissingSidecarReuploadsBoth(t *testing.T) { require.NoError(t, uploadDir.Delete(ctx, tarName+segmentIndexSuffix)) rec.reset() - second, err := uploadSnapshot(ctx, w, snap, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase, false) + second, err := uploadSnapshot(ctx, w, snap, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase, fixedV2(false)) require.NoError(t, err) assert.Equal(t, first.CodeSourcePath, second.CodeSourcePath) assert.Equal(t, map[string]bool{tarName: true, tarName + segmentIndexSuffix: true}, rec.names()) diff --git a/cmd/air/snapshot_v2.go b/cmd/air/snapshot_v2.go index f1e965c614e..1026f7091ea 100644 --- a/cmd/air/snapshot_v2.go +++ b/cmd/air/snapshot_v2.go @@ -33,6 +33,10 @@ import ( // codeSnapshotV2Capability is the config:validate capability that enables v2. const codeSnapshotV2Capability = "code_snapshot_v2" +// v2Decision reports whether the backend advertised v2. It blocks until config:validate has +// finished, so it is only called where the answer is needed. +type v2Decision func(ctx context.Context) (bool, error) + const ( v2Buckets = 16 v2ManifestFormat = "air-snapshot-manifest" @@ -93,15 +97,15 @@ func memberOf(bucket, k int) int { return bucket / (v2Buckets / k) } -// v2Header is the tar header for info with owner fields cleared. The owner names come from -// the local user database and would change ids between machines. +// v2Header is the tar header for info. It carries the same owner fields as v1, so extraction +// restores the same ownership. Ids therefore depend on the machine that packaged the tree, +// and another machine re-uploads once. func v2Header(info fs.FileInfo, name, linkTarget string) (*tar.Header, error) { hdr, err := tar.FileInfoHeader(info, linkTarget) if err != nil { return nil, err } hdr.Name = name - hdr.Uid, hdr.Gid, hdr.Uname, hdr.Gname = 0, 0, "", "" return hdr, nil } diff --git a/cmd/air/snapshot_v2_test.go b/cmd/air/snapshot_v2_test.go index 06d6a89cfae..4ca4e15986a 100644 --- a/cmd/air/snapshot_v2_test.go +++ b/cmd/air/snapshot_v2_test.go @@ -1,11 +1,16 @@ package aircmd import ( + "archive/tar" "bytes" "compress/gzip" + "context" "encoding/hex" "encoding/json" + "errors" "fmt" + "io" + "maps" "net/http" "os" "os/exec" @@ -107,22 +112,134 @@ func TestKForTarSize(t *testing.T) { // Same tree gives the same ids, and the ids are pinned so a change to the fragment encoding // is visible in review (it re-uploads every member once). -func TestV2IDsAreDeterministicAndPinned(t *testing.T) { - repo := writeV2Tree(t, v2TestTree(24)) - first := v2IDs(t, repo, v2Buckets) - second := v2IDs(t, repo, v2Buckets) - assert.Equal(t, first, second) - - var got []string - for id := range first { - got = append(got, id) +// The same tree gives the same ids, from one run to the next and from a copy in another +// directory. Ids are not pinned because owner fields are part of the fragment, so they depend +// on the user that packages the tree. +func TestV2IDsAreDeterministic(t *testing.T) { + first := v2IDs(t, writeV2Tree(t, v2TestTree(24)), v2Buckets) + assert.Equal(t, first, v2IDs(t, writeV2Tree(t, v2TestTree(24)), v2Buckets)) + assert.Equal(t, first, v2IDs(t, writeV2Tree(t, v2TestTree(24)), v2Buckets)) +} + +// fixedV2 is a v2Decision with a fixed answer. +func fixedV2(v2 bool) v2Decision { + return func(context.Context) (bool, error) { return v2, nil } +} + +// entryHeader is the part of a tar header that must match between v1 and v2. +type entryHeader struct { + Typeflag byte + Mode int64 + ModTime int64 + Uid, Gid int + Uname string + Gname string + Linkname string + Size int64 +} + +// tarHeaders reads every entry header from a tar stream. +func tarHeaders(t *testing.T, r io.Reader) map[string]entryHeader { + t.Helper() + out := map[string]entryHeader{} + tr := tar.NewReader(r) + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + return out + } + require.NoError(t, err) + out[hdr.Name] = entryHeader{ + Typeflag: hdr.Typeflag, + Mode: hdr.Mode, + ModTime: hdr.ModTime.UnixNano(), + Uid: hdr.Uid, + Gid: hdr.Gid, + Uname: hdr.Uname, + Gname: hdr.Gname, + Linkname: hdr.Linkname, + Size: hdr.Size, + } } - slices.Sort(got) - t.Logf("ids: %v", got) +} - // Pinned for the fixed tree above (fixed mtime and mode, owner fields cleared). - want := []string{"2030798903f5ffd2d9bfd594b90f00db", "42d7e05f72052c59572de52b8685133c", "5aded820cc397b719161ef29cce01977", "7cef810a2d0a0108a199cac9b530f96c", "910cc54d61649a4f83b22932506d9a35", "91972034b47e8679e3f9c36370d8d030", "ba2ce71feaf6e8bfcc3c316df9050830", "bd4f954cb0158d38f8bc36c6902ad8a5", "d151e658c32c0de18699b99889a77982", "efec3e984af63c246333c8e1c1f96a3d", "f2c7eda8bd574178784aabddd415a0b5", "f2fb48177b6de2b085280f2ee02ee6af", "fb1516bccc909166ff6dedb7da1f50b9", "fbf370ae26a25139242b0b1840db4974"} - assert.Equal(t, want, got) +// A v1 tarball and the v2 members of the same tree carry identical headers for every entry: +// name, mode, mtime, owner, type, link target and size. Only the packaging differs. +func TestV2AndV1EntryHeadersMatch(t *testing.T) { + repo := writeV2Tree(t, v2TestTree(60)) + require.NoError(t, os.Symlink("pkg0/mod0.py", filepath.Join(repo, "link.py"))) + files, err := snapshotFiles(t.Context(), repo, nil, false) + require.NoError(t, err) + + v1Path := filepath.Join(t.TempDir(), "v1.tar.gz") + _, err = createPlainTarball(t.Context(), repo, v1Path, files) + require.NoError(t, err) + v1Data, err := os.ReadFile(v1Path) + require.NoError(t, err) + gz, err := gzip.NewReader(bytes.NewReader(v1Data)) + require.NoError(t, err) + v1 := tarHeaders(t, gz) + + tmp := t.TempDir() + members, err := buildV2Members(t.Context(), tmp, repo, filepath.Base(repo), files, v2Buckets) + require.NoError(t, err) + v2 := map[string]entryHeader{} + for _, m := range members { + f, err := os.Open(m.fragment) + require.NoError(t, err) + maps.Copy(v2, tarHeaders(t, f)) + require.NoError(t, f.Close()) + } + + require.Len(t, v2, len(v1)) + for name, want := range v1 { + assert.Equal(t, want, v2[name], name) + } + assert.Equal(t, byte(tar.TypeSymlink), v1[path.Join(filepath.Base(repo), "link.py")].Typeflag) +} + +// The manifest has exactly the agreed fields, and the end object is last. +func TestV2ManifestSchema(t *testing.T) { + repo := writeV2Tree(t, v2TestTree(30)) + store := newV2Store(t, nil) + files, err := snapshotFiles(t.Context(), repo, nil, false) + require.NoError(t, err) + var result snapshotResult + require.NoError(t, uploadSnapshotV2(t.Context(), store.f, testSnapshotArtifactPath+"/.internal", repo, files, &result)) + + rc, err := store.f.Read(t.Context(), path.Base(result.CodeSourcePath)) + require.NoError(t, err) + defer rc.Close() + var manifest map[string]json.RawMessage + require.NoError(t, json.NewDecoder(rc).Decode(&manifest)) + keys := make([]string, 0, len(manifest)) + for k := range manifest { + keys = append(keys, k) + } + slices.Sort(keys) + assert.Equal(t, []string{"component", "format", "k", "members", "tar_size", "version"}, keys) + + var typed struct { + Format string `json:"format"` + Version int `json:"version"` + K int `json:"k"` + Members []v2Member `json:"members"` + } + raw, err := json.Marshal(manifest) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(raw, &typed)) + assert.Equal(t, "air-snapshot-manifest", typed.Format) + assert.Equal(t, 2, typed.Version) + require.NotEmpty(t, typed.Members) + + end, err := writeV2EndFragment(t.TempDir()) + require.NoError(t, err) + assert.Equal(t, end.ID, typed.Members[len(typed.Members)-1].ID, "end object is last") + for _, m := range typed.Members { + assert.Len(t, m.ID, 32) + assert.Positive(t, m.GzSize) + assert.Positive(t, m.TarSize) + } } // A one-file edit changes exactly one member id: the member holding the file. diff --git a/cmd/air/telemetry_test.go b/cmd/air/telemetry_test.go index 081af188483..1b46dba57a9 100644 --- a/cmd/air/telemetry_test.go +++ b/cmd/air/telemetry_test.go @@ -180,7 +180,7 @@ func TestSnapshotPackagingFailureMeasurements(t *testing.T) { // The commit is no longer available when git archive attempts to package it. result, err := uploadSnapshotTarball(t.Context(), w, newTestRepo(t), snapshotPlan{ mode: modeGitArchive, commitSHA: "missing-commit", isGitRepo: true, - }, testSnapshotArtifactPath, false) + }, testSnapshotArtifactPath, fixedV2(false)) require.Error(t, err) assert.Equal(t, new(modeGitArchive), result.PackagingMode) require.NotNil(t, result.PackagingDurationMs)