From 74f603e89f17c0614048bd201d3a171cdcf82e53 Mon Sep 17 00:00:00 2001 From: Steven Chen Date: Mon, 9 Nov 2026 07:28:39 +0000 Subject: [PATCH 1/4] air: write plain_tar code snapshots as segmented gzip members The plain_tar snapshot is now several independently compressed gzip members whose concatenated payload is one tar archive, so old launchers still extract it with a single `tar -xzf`. Members are balanced by bytes (~32 MiB, max 16) and compressed in parallel. An index sidecar (.idx.json) describes the member ranges and is uploaded before the tarball. The plain_tar key version is bumped so old single-member tarballs are not reused. Co-authored-by: Isaac --- cmd/air/runsubmit_test.go | 61 ++++++++ cmd/air/snapshot_cachekey.go | 2 +- cmd/air/snapshot_package.go | 48 +++---- cmd/air/snapshot_package_test.go | 12 +- cmd/air/snapshot_segment.go | 176 +++++++++++++++++++++++ cmd/air/snapshot_segment_test.go | 232 +++++++++++++++++++++++++++++++ cmd/air/snapshot_upload.go | 37 ++++- libs/tarpack/tarpack.go | 22 ++- libs/tarpack/tarpack_test.go | 19 +++ 9 files changed, 563 insertions(+), 46 deletions(-) create mode 100644 cmd/air/snapshot_segment.go create mode 100644 cmd/air/snapshot_segment_test.go diff --git a/cmd/air/runsubmit_test.go b/cmd/air/runsubmit_test.go index 441a1c12d26..a81f261bbb4 100644 --- a/cmd/air/runsubmit_test.go +++ b/cmd/air/runsubmit_test.go @@ -7,6 +7,7 @@ import ( "io/fs" "path" "path/filepath" + "slices" "strings" "sync" "sync/atomic" @@ -817,6 +818,66 @@ func testSidecarStore(t *testing.T, w *databricks.WorkspaceClient) (filer.Filer, const testSnapshotArtifactPath = "/Workspace/Users/tester@databricks.com/.air/repo_snapshots" +// A plain-tar snapshot uploads its index sidecar before the tarball, so a visible tarball +// always has its index. A cache hit writes neither. +func TestSubmitWorkloadPlainTarUploadsIndexBeforeTarball(t *testing.T) { + server := testserver.New(t) + t.Cleanup(server.Close) + + server.Handle("POST", "/api/2.2/jobs/runs/submit", func(req testserver.Request) any { + return jobs.SubmitRunResponse{RunId: 555} + }) + var writes []string + var index segmentIndex + server.Handle("POST", "/api/2.0/workspace-files/import-file/{path...}", func(req testserver.Request) any { + p := req.Vars["path"] + if strings.Contains(p, "/.air/repo_snapshots/") { + // The filer retries the import after creating a missing parent dir, so + // record each object's first write only. + name := path.Base(p) + if !slices.Contains(writes, name) { + writes = append(writes, name) + if strings.HasSuffix(name, segmentIndexSuffix) { + require.NoError(t, json.Unmarshal(req.Body, &index)) + } + } + } + return req.Workspace.WorkspaceFilesImportFile(p, req.Body, req.URL.Query().Get("overwrite") == "true") + }) + stubValidateConfig(server) + testserver.AddDefaultHandlers(server) + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "token"}) + require.NoError(t, err) + + repo := filepath.Join(t.TempDir(), "src") + writeRepoFile(t, repo, "train.py", "print()") + cfgPath := writeConfigFile(t, "run.yaml", minimalConfig+` +code_source: + type: snapshot + snapshot: + root_path: `+repo+` +`) + loaded, err := loadRunConfig(cfgPath) + require.NoError(t, err) + + ctx := cmdio.MockDiscard(t.Context()) + sidecarStore, sidecarBase := testSidecarStore(t, w) + first, err := uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase) + require.NoError(t, err) + + tarName := path.Base(first.CodeSourcePath) + assert.Equal(t, []string{tarName + segmentIndexSuffix, tarName}, writes) + assert.Equal(t, segmentFormat, index.Format) + assert.Equal(t, "src", index.Component) + require.NotNil(t, first.SizeBytes) + assert.Equal(t, *first.SizeBytes, index.Size) + assert.Len(t, index.Members, 1) + + _, err = uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase) + require.NoError(t, err) + assert.Len(t, writes, 2, "cache hit should write neither the index nor the tarball") +} + // A plain-tar (working-tree) snapshot is content-addressed by its file fingerprint // (working-tree metadata): submitting the same unchanged tree twice reuses the already-uploaded // tarball and skips the second upload, resolving to the identical remote path. diff --git a/cmd/air/snapshot_cachekey.go b/cmd/air/snapshot_cachekey.go index 5c2cf3f7b1d..7b278a3a8cf 100644 --- a/cmd/air/snapshot_cachekey.go +++ b/cmd/air/snapshot_cachekey.go @@ -20,7 +20,7 @@ const snapshotPackagingVersion = "v1" // with a git_archive key) and lets us invalidate it if the fingerprint scheme changes. // computePlainTarKey also folds in the shared snapshotPackagingVersion, so a // packaging-logic bump invalidates both modes' keys. -const plainTarKeyVersion = "plaintar-v2" +const plainTarKeyVersion = "plaintar-v3" // computePlainTarKey returns a content-addressed key for a working-tree snapshot: the // SHA-256 over every file's path, size, mtime, mode, and symlink target (sorted for diff --git a/cmd/air/snapshot_package.go b/cmd/air/snapshot_package.go index ca8a6942f5f..609c07950ad 100644 --- a/cmd/air/snapshot_package.go +++ b/cmd/air/snapshot_package.go @@ -12,7 +12,6 @@ import ( "strings" "github.com/databricks/cli/libs/tarpack" - "github.com/klauspost/pgzip" ) // Tar builders ported from cli/utils/snapshot.py. The git-ref path snapshots a commit @@ -50,46 +49,31 @@ func createGitArchiveSnapshot(ctx context.Context, git gitRepo, commitSHA, outpu return nil } -// createPlainTarball writes a gzipped tar of the given working-tree files to -// outputTarball. The archive preserves repoPath's directory name as the top-level -// entry. files come pre-resolved from snapshotFiles (.gitignore honored, .git and -// macOS AppleDouble excluded), so the caller can reuse the same listing to -// content-address the upload. -func createPlainTarball(ctx context.Context, repoPath, outputTarball string, files []snapshotFile) error { +// createPlainTarball writes the given working-tree files to outputTarball as a +// segmented tarball (see snapshot_segment.go) and returns its index. The archive +// preserves repoPath's directory name as the top-level entry. files come pre-resolved +// from snapshotFiles (.gitignore honored, .git and macOS AppleDouble excluded), so the +// caller can reuse the same listing to content-address the upload. +func createPlainTarball(ctx context.Context, repoPath, outputTarball string, files []snapshotFile) (segmentIndex, error) { dirName := filepath.Base(repoPath) + entries, sizes := plainTarEntries(repoPath, files) + return writeSegmentedTarball(ctx, dirName, outputTarball, entries, sizes, segmentTargetBytes, segmentMaxMembers) +} +// plainTarEntries returns the tar entries for files under repoPath's directory name, +// with each entry's estimated tar size for balancing members. +func plainTarEntries(repoPath string, files []snapshotFile) ([]tarpack.Entry, []int64) { + dirName := filepath.Base(repoPath) entries := make([]tarpack.Entry, len(files)) + sizes := make([]int64, len(files)) for i, file := range files { entries[i] = tarpack.Entry{ Name: filepath.ToSlash(filepath.Join(dirName, file.rel)), Path: filepath.Join(repoPath, file.rel), } + sizes[i] = file.tarSize() } - - out, err := os.Create(outputTarball) - if err != nil { - return fmt.Errorf("failed to create tarball: %w", err) - } - - // Gzip in parallel with klauspost/pgzip rather than a single-threaded writer: - // compression dominates packaging time on a large tree, and pgzip spreads it - // across cores. tarpack writes the tar in Go, so there is no `tar` subprocess - // (and no Windows colon-in-path issue a `tar -f` argument would hit). - gz, err := pgzip.NewWriterLevel(out, pgzip.DefaultCompression) - if err != nil { - out.Close() - return err - } - if err := tarpack.Write(ctxWriter{ctx, gz}, entries); err != nil { - gz.Close() - out.Close() - return fmt.Errorf("failed to create plain tarball: %w", err) - } - if err := gz.Close(); err != nil { - out.Close() - return fmt.Errorf("failed to finalize gzip: %w", err) - } - return out.Close() + return entries, sizes } // ctxWriter fails writes once ctx is done, so a cancelled submission (for diff --git a/cmd/air/snapshot_package_test.go b/cmd/air/snapshot_package_test.go index db42f634356..64a0aec5533 100644 --- a/cmd/air/snapshot_package_test.go +++ b/cmd/air/snapshot_package_test.go @@ -135,7 +135,7 @@ func TestCreatePlainTarball(t *testing.T) { out := filepath.Join(t.TempDir(), "snap.tar.gz") files, err := snapshotFiles(ctx, repo, nil, false) require.NoError(t, err) - require.NoError(t, createPlainTarball(ctx, repo, out, files)) + writePlainTarball(t, ctx, repo, out, files) dirName := filepath.Base(repo) entries := tarballEntries(t, out) @@ -155,7 +155,7 @@ func TestCreatePlainTarballStopsWhenCancelled(t *testing.T) { ctx, cancel := context.WithCancel(t.Context()) cancel() - err = createPlainTarball(ctx, repo, filepath.Join(t.TempDir(), "snap.tar.gz"), files) + _, err = createPlainTarball(ctx, repo, filepath.Join(t.TempDir(), "snap.tar.gz"), files) require.ErrorIs(t, err, context.Canceled) } @@ -169,7 +169,7 @@ func TestCreatePlainTarball_HonorsGitignore(t *testing.T) { out := filepath.Join(t.TempDir(), "snap.tar.gz") files, err := snapshotFiles(ctx, repo, nil, false) require.NoError(t, err) - require.NoError(t, createPlainTarball(ctx, repo, out, files)) + writePlainTarball(t, ctx, repo, out, files) dirName := filepath.Base(repo) entries := tarballEntries(t, out) @@ -186,7 +186,7 @@ func TestCreatePlainTarball_IncludePaths(t *testing.T) { out := filepath.Join(t.TempDir(), "snap.tar.gz") files, err := snapshotFiles(ctx, repo, []string{"src"}, false) require.NoError(t, err) - require.NoError(t, createPlainTarball(ctx, repo, out, files)) + writePlainTarball(t, ctx, repo, out, files) dirName := filepath.Base(repo) entries := tarballEntries(t, out) @@ -208,7 +208,7 @@ func TestCreatePlainTarball_HonorsNestedGitignoreAndNegation(t *testing.T) { out := filepath.Join(t.TempDir(), "snap.tar.gz") files, err := snapshotFiles(t.Context(), repo, nil, false) require.NoError(t, err) - require.NoError(t, createPlainTarball(t.Context(), repo, out, files)) + writePlainTarball(t, t.Context(), repo, out, files) dirName := filepath.Base(repo) entries := tarballEntries(t, out) @@ -230,7 +230,7 @@ func TestCreatePlainTarball_SkipsDeletedTrackedFiles(t *testing.T) { out := filepath.Join(t.TempDir(), "snap.tar.gz") files, err := snapshotFiles(t.Context(), repo, nil, true) require.NoError(t, err) - require.NoError(t, createPlainTarball(t.Context(), repo, out, files)) + writePlainTarball(t, t.Context(), repo, out, files) dirName := filepath.Base(repo) entries := tarballEntries(t, out) diff --git a/cmd/air/snapshot_segment.go b/cmd/air/snapshot_segment.go new file mode 100644 index 00000000000..e15994bf0ec --- /dev/null +++ b/cmd/air/snapshot_segment.go @@ -0,0 +1,176 @@ +package aircmd + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "runtime" + + "github.com/databricks/cli/libs/tarpack" + "github.com/klauspost/pgzip" + "golang.org/x/sync/errgroup" +) + +// A segmented snapshot is a .tar.gz made of several concatenated gzip members whose +// uncompressed payload is one tar archive. Plain `tar -xzf` reads it as a single +// stream, so launchers that predate the index keep working. The index sidecar +// (.idx.json) lets a launcher decompress members in parallel. + +const ( + segmentFormat = "air-segmented-tar-gz" + segmentFormatVersion = 1 + segmentIndexSuffix = ".idx.json" + segmentTargetBytes = 32 << 20 + segmentMaxMembers = 16 + + tarBlockSize = 512 +) + +// segmentIndex describes the members of a segmented tarball. Members are contiguous +// and cover [0, Size). +type segmentIndex struct { + Format string `json:"format"` + Version int `json:"version"` + Component string `json:"component"` + Size int64 `json:"size"` + Members []segmentMember `json:"members"` +} + +// segmentMember is the byte range of one gzip member in the tarball. +type segmentMember struct { + Offset int64 `json:"offset"` + Length int64 `json:"length"` +} + +// tarSize estimates the uncompressed tar bytes the entry occupies (header plus padded +// data), matching what tarpack writes for regular files and symlinks. Other file types +// are skipped by tarpack and take no space. +func (f snapshotFile) tarSize() int64 { + mode := os.FileMode(f.mode) + switch { + case mode.IsRegular(): + return tarBlockSize + (f.size+tarBlockSize-1)/tarBlockSize*tarBlockSize + case mode&os.ModeSymlink != 0: + return tarBlockSize + } + return 0 +} + +// planSegments returns the index of the first entry of each member, given each +// entry's tar size. Members are contiguous and balanced by bytes: the member count is +// ceil(total/targetBytes) capped at maxMembers, and an entry goes to the member +// containing its starting offset. There is always at least one member. +func planSegments(sizes []int64, targetBytes int64, maxMembers int) []int { + var total int64 + for _, s := range sizes { + total += s + } + if total == 0 { + return []int{0} + } + n := max(1, min(maxMembers, int((total+targetBytes-1)/targetBytes))) + + starts := []int{0} + var offset int64 + current := 0 + for i, s := range sizes { + member := int(offset * int64(n) / total) + if member != current { + starts = append(starts, i) + current = member + } + offset += s + } + return starts +} + +// writeSegmentedTarball writes entries to outputTarball as independently compressed +// gzip members, compressed in parallel, and returns the index of their byte ranges. +// Only the last member carries the tar end-of-archive marker. sizes[i] is the +// estimated tar size of entries[i] and is used only for balancing. +func writeSegmentedTarball(ctx context.Context, component, outputTarball string, entries []tarpack.Entry, sizes []int64, targetBytes int64, maxMembers int) (segmentIndex, error) { + starts := planSegments(sizes, targetBytes, maxMembers) + + tmp, err := os.MkdirTemp("", "air-snapshot-members-") + if err != nil { + return segmentIndex{}, fmt.Errorf("failed to create snapshot members dir: %w", err) + } + defer os.RemoveAll(tmp) + + memberPaths := make([]string, len(starts)) + g, gctx := errgroup.WithContext(ctx) + g.SetLimit(runtime.NumCPU()) + for i, start := range starts { + end := len(entries) + if i+1 < len(starts) { + end = starts[i+1] + } + memberPaths[i] = filepath.Join(tmp, fmt.Sprintf("member-%d", i)) + last := i == len(starts)-1 + g.Go(func() error { + return compressMember(gctx, memberPaths[i], entries[start:end], last) + }) + } + if err := g.Wait(); err != nil { + return segmentIndex{}, err + } + + out, err := os.Create(outputTarball) + if err != nil { + return segmentIndex{}, fmt.Errorf("failed to create tarball: %w", err) + } + defer out.Close() + + index := segmentIndex{Format: segmentFormat, Version: segmentFormatVersion, Component: component} + for _, p := range memberPaths { + length, err := appendFile(out, p) + if err != nil { + return segmentIndex{}, err + } + index.Members = append(index.Members, segmentMember{Offset: index.Size, Length: length}) + index.Size += length + } + if err := out.Close(); err != nil { + return segmentIndex{}, fmt.Errorf("failed to finalize tarball: %w", err) + } + return index, nil +} + +// compressMember writes entries as one gzip member at path. Members other than the +// last omit the tar end-of-archive marker. +func compressMember(ctx context.Context, path string, entries []tarpack.Entry, last bool) error { + f, err := os.Create(path) + if err != nil { + return err + } + gz, err := pgzip.NewWriterLevel(ctxWriter{ctx, f}, pgzip.DefaultCompression) + if err != nil { + f.Close() + return err + } + write := tarpack.WriteFragment + if last { + write = tarpack.Write + } + if err := write(gz, entries); err != nil { + f.Close() + return fmt.Errorf("failed to create plain tarball: %w", err) + } + if err := gz.Close(); err != nil { + f.Close() + return fmt.Errorf("failed to finalize gzip: %w", err) + } + return f.Close() +} + +// appendFile copies the file at path onto the end of w and returns the bytes copied. +func appendFile(w io.Writer, path string) (int64, error) { + f, err := os.Open(path) + if err != nil { + return 0, err + } + defer f.Close() + return io.Copy(w, f) +} diff --git a/cmd/air/snapshot_segment_test.go b/cmd/air/snapshot_segment_test.go new file mode 100644 index 00000000000..94489552ebb --- /dev/null +++ b/cmd/air/snapshot_segment_test.go @@ -0,0 +1,232 @@ +package aircmd + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// writePlainTarball packages files with createPlainTarball and fails the test on error. +func writePlainTarball(t *testing.T, ctx context.Context, repo, out string, files []snapshotFile) { + t.Helper() + _, err := createPlainTarball(ctx, repo, out, files) + require.NoError(t, err) +} + +// readSingleStream reads the .tar.gz as one gzip stream and returns the entry names, +// as an old launcher's `tar -xzf` does. It fails on any error other than the clean +// end of the archive. +func readSingleStream(t *testing.T, b []byte) []string { + t.Helper() + gz, err := gzip.NewReader(bytes.NewReader(b)) + require.NoError(t, err) + var names []string + tr := tar.NewReader(gz) + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + return names + } + require.NoError(t, err) + names = append(names, hdr.Name) + _, err = io.Copy(io.Discard, tr) + require.NoError(t, err) + } +} + +// readMemberFragment decodes one member on its own and returns its entry names. +// Members other than the last end without the end-of-archive marker, so a clean EOF +// on a header boundary is the expected end. +func readMemberFragment(t *testing.T, member []byte) []string { + t.Helper() + gz, err := gzip.NewReader(bytes.NewReader(member)) + require.NoError(t, err) + var names []string + tr := tar.NewReader(gz) + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + return names + } + require.NoError(t, err) + names = append(names, hdr.Name) + _, err = io.Copy(io.Discard, tr) + require.NoError(t, err) + } +} + +// makeSegmentRepo creates n files of growing size under a repo directory and returns +// the repo path and its snapshot file listing. +func makeSegmentRepo(t *testing.T, n int) (string, []snapshotFile) { + t.Helper() + repo := filepath.Join(t.TempDir(), "code") + for i := range n { + writeRepoFile(t, repo, fmt.Sprintf("pkg%d/file%d.py", i%3, i), strings.Repeat(fmt.Sprintf("x = %d\n", i), 50*i+1)) + } + files, err := snapshotFiles(t.Context(), repo, nil, false) + require.NoError(t, err) + return repo, files +} + +func TestPlanSegments(t *testing.T) { + tests := []struct { + name string + sizes []int64 + target int64 + maxMembers int + want []int + }{ + {name: "no entries is one member", sizes: nil, target: 100, maxMembers: 16, want: []int{0}}, + {name: "all zero-size entries is one member", sizes: []int64{0, 0}, target: 100, maxMembers: 16, want: []int{0}}, + {name: "under target is one member", sizes: []int64{40, 40}, target: 100, maxMembers: 16, want: []int{0}}, + {name: "balanced by bytes", sizes: []int64{50, 50, 50, 50}, target: 100, maxMembers: 16, want: []int{0, 2}}, + {name: "big entry is not split", sizes: []int64{300, 10}, target: 100, maxMembers: 16, want: []int{0, 1}}, + {name: "capped at max members", sizes: []int64{10, 10, 10, 10, 10, 10}, target: 10, maxMembers: 2, want: []int{0, 3}}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, planSegments(tc.sizes, tc.target, tc.maxMembers)) + }) + } +} + +// Old launchers read the whole .tar.gz as one gzip stream: it must yield exactly the +// expected entries, even with several members. +func TestSegmentedTarballReadsAsSingleStream(t *testing.T) { + ctx := t.Context() + repo, files := makeSegmentRepo(t, 30) + entries, sizes := plainTarEntries(repo, files) + out := filepath.Join(t.TempDir(), "snap.tar.gz") + index, err := writeSegmentedTarball(ctx, filepath.Base(repo), out, entries, sizes, 4096, 16) + require.NoError(t, err) + require.Greater(t, len(index.Members), 1) + + data, err := os.ReadFile(out) + require.NoError(t, err) + var want []string + for _, e := range entries { + want = append(want, e.Name) + } + assert.ElementsMatch(t, want, readSingleStream(t, data)) +} + +// Each member decodes independently to a valid tar fragment, and the union of the +// members is the full set of entries in order. +func TestSegmentedTarballMembersDecodeIndependently(t *testing.T) { + ctx := t.Context() + repo, files := makeSegmentRepo(t, 30) + entries, sizes := plainTarEntries(repo, files) + out := filepath.Join(t.TempDir(), "snap.tar.gz") + index, err := writeSegmentedTarball(ctx, filepath.Base(repo), out, entries, sizes, 4096, 16) + require.NoError(t, err) + require.Greater(t, len(index.Members), 1) + + data, err := os.ReadFile(out) + require.NoError(t, err) + var union []string + for _, m := range index.Members { + union = append(union, readMemberFragment(t, data[m.Offset:m.Offset+m.Length])...) + } + var want []string + for _, e := range entries { + want = append(want, e.Name) + } + assert.Equal(t, want, union) +} + +// The index matches the tarball: members are contiguous, cover [0, size), size equals +// the file size, and every entry sits under the component directory. +func TestSegmentedTarballIndexContract(t *testing.T) { + ctx := t.Context() + repo, files := makeSegmentRepo(t, 30) + entries, sizes := plainTarEntries(repo, files) + out := filepath.Join(t.TempDir(), "snap.tar.gz") + index, err := writeSegmentedTarball(ctx, filepath.Base(repo), out, entries, sizes, 4096, 16) + require.NoError(t, err) + + info, err := os.Stat(out) + require.NoError(t, err) + assert.Equal(t, segmentFormat, index.Format) + assert.Equal(t, segmentFormatVersion, index.Version) + assert.Equal(t, "code", index.Component) + assert.Equal(t, info.Size(), index.Size) + + var next int64 + for _, m := range index.Members { + assert.Equal(t, next, m.Offset) + assert.Positive(t, m.Length) + next += m.Length + } + assert.Equal(t, index.Size, next) + + data, err := os.ReadFile(out) + require.NoError(t, err) + for _, name := range readSingleStream(t, data) { + assert.True(t, strings.HasPrefix(name, "code/"), name) + } +} + +// Only the last member carries the end-of-archive marker. +func TestSegmentedTarballEndMarkerOnlyInLastMember(t *testing.T) { + ctx := t.Context() + repo, files := makeSegmentRepo(t, 30) + entries, sizes := plainTarEntries(repo, files) + out := filepath.Join(t.TempDir(), "snap.tar.gz") + index, err := writeSegmentedTarball(ctx, filepath.Base(repo), out, entries, sizes, 4096, 16) + require.NoError(t, err) + require.Greater(t, len(index.Members), 1) + + data, err := os.ReadFile(out) + require.NoError(t, err) + for i, m := range index.Members { + gz, err := gzip.NewReader(bytes.NewReader(data[m.Offset : m.Offset+m.Length])) + require.NoError(t, err) + payload, err := io.ReadAll(gz) + require.NoError(t, err) + trailer := make([]byte, 1024) + endsWithMarker := bytes.Equal(payload[len(payload)-1024:], trailer) + assert.Equal(t, i == len(index.Members)-1, endsWithMarker, "member %d", i) + } +} + +// A small repo packages as one member, and the tarball is a plain single-member archive. +func TestCreatePlainTarballSmallRepoIsOneMember(t *testing.T) { + repo, files := makeSegmentRepo(t, 3) + out := filepath.Join(t.TempDir(), "snap.tar.gz") + index, err := createPlainTarball(t.Context(), repo, out, files) + require.NoError(t, err) + assert.Len(t, index.Members, 1) + assert.Equal(t, []segmentMember{{Offset: 0, Length: index.Size}}, index.Members) +} + +// Same input yields byte-identical tarballs and indexes, which the content-addressed +// cache key depends on. +func TestCreatePlainTarballIsDeterministic(t *testing.T) { + repo, files := makeSegmentRepo(t, 30) + dir := t.TempDir() + first := filepath.Join(dir, "first.tar.gz") + second := filepath.Join(dir, "second.tar.gz") + entries, sizes := plainTarEntries(repo, files) + indexA, err := writeSegmentedTarball(t.Context(), filepath.Base(repo), first, entries, sizes, 4096, 16) + require.NoError(t, err) + indexB, err := writeSegmentedTarball(t.Context(), filepath.Base(repo), second, entries, sizes, 4096, 16) + require.NoError(t, err) + + a, err := os.ReadFile(first) + require.NoError(t, err) + b, err := os.ReadFile(second) + require.NoError(t, err) + assert.Equal(t, a, b) + assert.Equal(t, indexA, indexB) +} diff --git a/cmd/air/snapshot_upload.go b/cmd/air/snapshot_upload.go index 1d4e9a533b8..1af77c6d52d 100644 --- a/cmd/air/snapshot_upload.go +++ b/cmd/air/snapshot_upload.go @@ -3,6 +3,7 @@ package aircmd import ( "bytes" "context" + "encoding/json" "errors" "fmt" "io/fs" @@ -137,13 +138,18 @@ func snapshotTarName(ctx context.Context, repoPath string, plan snapshotPlan) (s } // packageSnapshot writes the snapshot to tarball per the resolved plan: `git archive` -// of the pinned commit for git_archive, else a plain tar of the pre-listed working-tree -// files (nil for git_archive). -func packageSnapshot(ctx context.Context, repoPath string, plan snapshotPlan, files []snapshotFile, tarball string) error { +// of the pinned commit for git_archive, else a segmented plain tar of the pre-listed +// working-tree files. It returns the segment index for plain_tar and nil for +// git_archive, which has no sidecar yet. +func packageSnapshot(ctx context.Context, repoPath string, plan snapshotPlan, files []snapshotFile, tarball string) (*segmentIndex, error) { if plan.mode == modeGitArchive { - return createGitArchiveSnapshot(ctx, newGitRepo(repoPath), plan.commitSHA, tarball, filepath.Base(repoPath), plan.includePaths, plan.subtreePrefix) + return nil, createGitArchiveSnapshot(ctx, newGitRepo(repoPath), plan.commitSHA, tarball, filepath.Base(repoPath), plan.includePaths, plan.subtreePrefix) } - return createPlainTarball(ctx, repoPath, tarball, files) + index, err := createPlainTarball(ctx, repoPath, tarball, files) + if err != nil { + return nil, err + } + return &index, nil } // airArtifactInternalDir mirrors bundle/libraries.InternalDirName: CLI-uploaded @@ -192,7 +198,7 @@ func uploadSnapshotTarball(ctx context.Context, w *databricks.WorkspaceClient, r tarball := filepath.Join(tmp, tarName) packagingStart := time.Now() - err = packageSnapshot(ctx, repoPath, plan, files, tarball) + index, err := packageSnapshot(ctx, repoPath, plan, files, tarball) result.PackagingDurationMs = new(time.Since(packagingStart).Milliseconds()) if err != nil { return result, err @@ -212,6 +218,12 @@ func uploadSnapshotTarball(ctx context.Context, w *databricks.WorkspaceClient, r } cmdio.LogProgress(ctx, fmt.Sprintf("Uploading %s...", tarName)) uploadStart := time.Now() + // The sidecar goes first so a tarball is never visible without its index. + if index != nil { + if err := uploadSegmentIndex(ctx, f, tarName, index); err != nil { + return result, err + } + } err = f.Write(ctx, tarName, file, filer.OverwriteIfExists, filer.CreateParentDirectories) result.UploadDurationMs = new(time.Since(uploadStart).Milliseconds()) if err != nil { @@ -221,6 +233,19 @@ func uploadSnapshotTarball(ctx context.Context, w *databricks.WorkspaceClient, r return result, nil } +// uploadSegmentIndex writes the index sidecar for tarName next to it in the store. +func uploadSegmentIndex(ctx context.Context, store filer.Filer, tarName string, index *segmentIndex) error { + data, err := json.Marshal(index) + if err != nil { + return fmt.Errorf("failed to encode snapshot index: %w", err) + } + err = store.Write(ctx, tarName+segmentIndexSuffix, bytes.NewReader(data), filer.OverwriteIfExists, filer.CreateParentDirectories) + if err != nil { + return fmt.Errorf("failed to upload snapshot index for %s: %w", tarName, err) + } + return nil +} + // snapshotUploadFiler returns a filer rooted at /.internal plus that // upload path. It routes /Volumes paths to a Files API filer and everything else to a // workspace files filer, matching bundle artifact-upload resolution without depending diff --git a/libs/tarpack/tarpack.go b/libs/tarpack/tarpack.go index 963988c0858..19de98d3248 100644 --- a/libs/tarpack/tarpack.go +++ b/libs/tarpack/tarpack.go @@ -29,12 +29,32 @@ type Entry struct { // matches what `tar` stored for the same file. func Write(w io.Writer, entries []Entry) error { tw := tar.NewWriter(w) + if err := writeEntries(tw, entries); err != nil { + return err + } + return tw.Close() +} + +// WriteFragment is Write without the end-of-archive marker. Concatenating +// fragments and then one Write-finalized archive yields a single tar stream, which +// lets callers split an archive into independently compressed parts. Each fragment +// ends on an entry boundary, so it can be read alone as a tar stream without its +// end marker. +func WriteFragment(w io.Writer, entries []Entry) error { + tw := tar.NewWriter(w) + if err := writeEntries(tw, entries); err != nil { + return err + } + return tw.Flush() +} + +func writeEntries(tw *tar.Writer, entries []Entry) error { for _, e := range entries { if err := writeEntry(tw, e); err != nil { return err } } - return tw.Close() + return nil } func writeEntry(tw *tar.Writer, e Entry) error { diff --git a/libs/tarpack/tarpack_test.go b/libs/tarpack/tarpack_test.go index 3841e779ac0..a3ccebd5989 100644 --- a/libs/tarpack/tarpack_test.go +++ b/libs/tarpack/tarpack_test.go @@ -96,6 +96,25 @@ func TestWriteSymlink(t *testing.T) { assert.Empty(t, got["pkg/link"].content) } +func TestWriteFragmentOmitsEndMarker(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "a.txt"), []byte("alpha"), 0o644)) + entries := []Entry{{Name: "pkg/a.txt", Path: filepath.Join(dir, "a.txt")}} + + var full, fragment bytes.Buffer + require.NoError(t, Write(&full, entries)) + require.NoError(t, WriteFragment(&fragment, entries)) + + // The fragment is the full archive minus the two zero end blocks. + assert.Equal(t, full.Bytes()[:fragment.Len()], fragment.Bytes()) + assert.Equal(t, 1024, full.Len()-fragment.Len()) + assert.Equal(t, make([]byte, 1024), full.Bytes()[fragment.Len():]) + + // Read as a standalone stream, the fragment yields its entry and then a clean EOF. + got := readTar(t, fragment.Bytes()) + assert.Equal(t, "alpha", got["pkg/a.txt"].content) +} + func TestWriteSkipsIrregularAndMissingIsError(t *testing.T) { dir := t.TempDir() From 3395a44d88c70cf1b0c5d0282558f56ad17c5112 Mon Sep 17 00:00:00 2001 From: Steven Chen Date: Mon, 9 Nov 2026 08:24:14 +0000 Subject: [PATCH 2/4] air: upload segmented snapshot objects concurrently and write members in place The index sidecar and tarball upload concurrently, and a cache hit for plain_tar requires both objects to exist. Compressed members are written directly into the output in order, instead of through a temp file and a concatenating copy. Co-authored-by: Isaac --- cmd/air/runsubmit_test.go | 71 +--------- cmd/air/snapshot_segment.go | 89 ++++++------- cmd/air/snapshot_segment_test.go | 216 +++++++++++++------------------ cmd/air/snapshot_upload.go | 47 +++++-- cmd/air/snapshot_upload_test.go | 143 ++++++++++++++++++++ 5 files changed, 316 insertions(+), 250 deletions(-) create mode 100644 cmd/air/snapshot_upload_test.go diff --git a/cmd/air/runsubmit_test.go b/cmd/air/runsubmit_test.go index a81f261bbb4..98bd7f9a4e0 100644 --- a/cmd/air/runsubmit_test.go +++ b/cmd/air/runsubmit_test.go @@ -7,7 +7,6 @@ import ( "io/fs" "path" "path/filepath" - "slices" "strings" "sync" "sync/atomic" @@ -818,66 +817,6 @@ func testSidecarStore(t *testing.T, w *databricks.WorkspaceClient) (filer.Filer, const testSnapshotArtifactPath = "/Workspace/Users/tester@databricks.com/.air/repo_snapshots" -// A plain-tar snapshot uploads its index sidecar before the tarball, so a visible tarball -// always has its index. A cache hit writes neither. -func TestSubmitWorkloadPlainTarUploadsIndexBeforeTarball(t *testing.T) { - server := testserver.New(t) - t.Cleanup(server.Close) - - server.Handle("POST", "/api/2.2/jobs/runs/submit", func(req testserver.Request) any { - return jobs.SubmitRunResponse{RunId: 555} - }) - var writes []string - var index segmentIndex - server.Handle("POST", "/api/2.0/workspace-files/import-file/{path...}", func(req testserver.Request) any { - p := req.Vars["path"] - if strings.Contains(p, "/.air/repo_snapshots/") { - // The filer retries the import after creating a missing parent dir, so - // record each object's first write only. - name := path.Base(p) - if !slices.Contains(writes, name) { - writes = append(writes, name) - if strings.HasSuffix(name, segmentIndexSuffix) { - require.NoError(t, json.Unmarshal(req.Body, &index)) - } - } - } - return req.Workspace.WorkspaceFilesImportFile(p, req.Body, req.URL.Query().Get("overwrite") == "true") - }) - stubValidateConfig(server) - testserver.AddDefaultHandlers(server) - w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "token"}) - require.NoError(t, err) - - repo := filepath.Join(t.TempDir(), "src") - writeRepoFile(t, repo, "train.py", "print()") - cfgPath := writeConfigFile(t, "run.yaml", minimalConfig+` -code_source: - type: snapshot - snapshot: - root_path: `+repo+` -`) - loaded, err := loadRunConfig(cfgPath) - require.NoError(t, err) - - ctx := cmdio.MockDiscard(t.Context()) - sidecarStore, sidecarBase := testSidecarStore(t, w) - first, err := uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase) - require.NoError(t, err) - - tarName := path.Base(first.CodeSourcePath) - assert.Equal(t, []string{tarName + segmentIndexSuffix, tarName}, writes) - assert.Equal(t, segmentFormat, index.Format) - assert.Equal(t, "src", index.Component) - require.NotNil(t, first.SizeBytes) - assert.Equal(t, *first.SizeBytes, index.Size) - assert.Len(t, index.Members, 1) - - _, err = uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase) - require.NoError(t, err) - assert.Len(t, writes, 2, "cache hit should write neither the index nor the tarball") -} - // A plain-tar (working-tree) snapshot is content-addressed by its file fingerprint // (working-tree metadata): submitting the same unchanged tree twice reuses the already-uploaded // tarball and skips the second upload, resolving to the identical remote path. @@ -893,11 +832,11 @@ func TestSubmitWorkloadPlainTarContentAddressed(t *testing.T) { // path) is what proves the skip: both submits resolve to the same content-addressed // name, so a set could not tell a skipped second submit from one that re-uploaded to // that same path. - snapshotUploads := 0 + var snapshotUploads atomic.Int32 server.Handle("POST", "/api/2.0/workspace-files/import-file/{path...}", func(req testserver.Request) any { p := req.Vars["path"] if strings.Contains(p, "/.air/repo_snapshots/") { - snapshotUploads++ + snapshotUploads.Add(1) } return req.Workspace.WorkspaceFilesImportFile(p, req.Body, req.URL.Query().Get("overwrite") == "true") }) @@ -925,8 +864,8 @@ code_source: sidecarStore, sidecarBase := testSidecarStore(t, w) first, err := uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase) require.NoError(t, err) - require.NotZero(t, snapshotUploads, "first submit should upload the tarball") - afterFirst := snapshotUploads + require.NotZero(t, snapshotUploads.Load(), "first submit should upload the tarball") + afterFirst := snapshotUploads.Load() second, err := uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase) require.NoError(t, err) @@ -938,7 +877,7 @@ code_source: // Same unchanged tree → identical remote path, and the second submit moved no bytes: // zero new import-file calls (a real skip), not a re-upload to the same name. assert.Equal(t, first.CodeSourcePath, second.CodeSourcePath) - assert.Equal(t, afterFirst, snapshotUploads, "unchanged plain_tar should skip the second upload") + assert.Equal(t, afterFirst, snapshotUploads.Load(), "unchanged plain_tar should skip the second upload") require.NotNil(t, first.SizeBytes) assert.Positive(t, *first.SizeBytes) assert.Nil(t, second.SizeBytes) diff --git a/cmd/air/snapshot_segment.go b/cmd/air/snapshot_segment.go index e15994bf0ec..97f61c5fb50 100644 --- a/cmd/air/snapshot_segment.go +++ b/cmd/air/snapshot_segment.go @@ -1,12 +1,11 @@ package aircmd import ( + "bytes" "context" "fmt" "io" "os" - "path/filepath" - "runtime" "github.com/databricks/cli/libs/tarpack" "github.com/klauspost/pgzip" @@ -87,90 +86,80 @@ func planSegments(sizes []int64, targetBytes int64, maxMembers int) []int { } // writeSegmentedTarball writes entries to outputTarball as independently compressed -// gzip members, compressed in parallel, and returns the index of their byte ranges. -// Only the last member carries the tar end-of-archive marker. sizes[i] is the -// estimated tar size of entries[i] and is used only for balancing. +// gzip members and returns the index of their byte ranges. sizes[i] is the estimated +// tar size of entries[i] and is used only for balancing. func writeSegmentedTarball(ctx context.Context, component, outputTarball string, entries []tarpack.Entry, sizes []int64, targetBytes int64, maxMembers int) (segmentIndex, error) { starts := planSegments(sizes, targetBytes, maxMembers) - - tmp, err := os.MkdirTemp("", "air-snapshot-members-") - if err != nil { - return segmentIndex{}, fmt.Errorf("failed to create snapshot members dir: %w", err) - } - defer os.RemoveAll(tmp) - - memberPaths := make([]string, len(starts)) - g, gctx := errgroup.WithContext(ctx) - g.SetLimit(runtime.NumCPU()) - for i, start := range starts { + return writeSegments(ctx, component, outputTarball, len(starts), func(ctx context.Context, i int, w io.Writer) error { end := len(entries) if i+1 < len(starts) { end = starts[i+1] } - memberPaths[i] = filepath.Join(tmp, fmt.Sprintf("member-%d", i)) - last := i == len(starts)-1 - g.Go(func() error { - return compressMember(gctx, memberPaths[i], entries[start:end], last) - }) - } - if err := g.Wait(); err != nil { - return segmentIndex{}, err - } + return writeMemberTar(ctx, w, entries[starts[i]:end], i == len(starts)-1) + }) +} +// writeSegments runs writeMember for n members concurrently, each into its own buffer, and +// appends the buffers to outputTarball in member order as they complete. Compressed +// members are held in memory until their turn, so peak memory is about the compressed +// size of the tarball, and no copy goes through a temp file. +func writeSegments(ctx context.Context, component, outputTarball string, n int, writeMember func(ctx context.Context, i int, w io.Writer) error) (segmentIndex, error) { out, err := os.Create(outputTarball) if err != nil { return segmentIndex{}, fmt.Errorf("failed to create tarball: %w", err) } defer out.Close() + ctx, cancel := context.WithCancel(ctx) + defer cancel() + bufs := make([]*bytes.Buffer, n) + done := make([]chan struct{}, n) + g, gctx := errgroup.WithContext(ctx) + for i := range n { + bufs[i] = &bytes.Buffer{} + done[i] = make(chan struct{}) + g.Go(func() error { + defer close(done[i]) + return writeMember(gctx, i, bufs[i]) + }) + } + index := segmentIndex{Format: segmentFormat, Version: segmentFormatVersion, Component: component} - for _, p := range memberPaths { - length, err := appendFile(out, p) + for i := range n { + <-done[i] + length, err := bufs[i].WriteTo(out) if err != nil { - return segmentIndex{}, err + return segmentIndex{}, fmt.Errorf("failed to write snapshot member %d: %w", i, err) } + bufs[i] = nil index.Members = append(index.Members, segmentMember{Offset: index.Size, Length: length}) index.Size += length } + if err := g.Wait(); err != nil { + return segmentIndex{}, err + } if err := out.Close(); err != nil { return segmentIndex{}, fmt.Errorf("failed to finalize tarball: %w", err) } return index, nil } -// compressMember writes entries as one gzip member at path. Members other than the -// last omit the tar end-of-archive marker. -func compressMember(ctx context.Context, path string, entries []tarpack.Entry, last bool) error { - f, err := os.Create(path) +// writeMemberTar writes entries to w as one gzip member. Members other than the last +// omit the tar end-of-archive marker. +func writeMemberTar(ctx context.Context, w io.Writer, entries []tarpack.Entry, last bool) error { + gz, err := pgzip.NewWriterLevel(ctxWriter{ctx, w}, pgzip.DefaultCompression) if err != nil { return err } - gz, err := pgzip.NewWriterLevel(ctxWriter{ctx, f}, pgzip.DefaultCompression) - if err != nil { - f.Close() - return err - } write := tarpack.WriteFragment if last { write = tarpack.Write } if err := write(gz, entries); err != nil { - f.Close() return fmt.Errorf("failed to create plain tarball: %w", err) } if err := gz.Close(); err != nil { - f.Close() return fmt.Errorf("failed to finalize gzip: %w", err) } - return f.Close() -} - -// appendFile copies the file at path onto the end of w and returns the bytes copied. -func appendFile(w io.Writer, path string) (int64, error) { - f, err := os.Open(path) - if err != nil { - return 0, err - } - defer f.Close() - return io.Copy(w, f) + return nil } diff --git a/cmd/air/snapshot_segment_test.go b/cmd/air/snapshot_segment_test.go index 94489552ebb..7465911cad1 100644 --- a/cmd/air/snapshot_segment_test.go +++ b/cmd/air/snapshot_segment_test.go @@ -13,6 +13,7 @@ import ( "strings" "testing" + "github.com/databricks/cli/libs/tarpack" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -24,15 +25,12 @@ func writePlainTarball(t *testing.T, ctx context.Context, repo, out string, file require.NoError(t, err) } -// readSingleStream reads the .tar.gz as one gzip stream and returns the entry names, -// as an old launcher's `tar -xzf` does. It fails on any error other than the clean -// end of the archive. -func readSingleStream(t *testing.T, b []byte) []string { +// tarNames reads a tar stream to its end and returns the entry names. A clean EOF is +// accepted at an entry boundary, so it also reads a member fragment with no end marker. +func tarNames(t *testing.T, r io.Reader) []string { t.Helper() - gz, err := gzip.NewReader(bytes.NewReader(b)) - require.NoError(t, err) var names []string - tr := tar.NewReader(gz) + tr := tar.NewReader(r) for { hdr, err := tr.Next() if errors.Is(err, io.EOF) { @@ -45,38 +43,39 @@ func readSingleStream(t *testing.T, b []byte) []string { } } -// readMemberFragment decodes one member on its own and returns its entry names. -// Members other than the last end without the end-of-archive marker, so a clean EOF -// on a header boundary is the expected end. -func readMemberFragment(t *testing.T, member []byte) []string { - t.Helper() - gz, err := gzip.NewReader(bytes.NewReader(member)) - require.NoError(t, err) - var names []string - tr := tar.NewReader(gz) - for { - hdr, err := tr.Next() - if errors.Is(err, io.EOF) { - return names - } - require.NoError(t, err) - names = append(names, hdr.Name) - _, err = io.Copy(io.Discard, tr) - require.NoError(t, err) - } +// segmentFixture is a repo packaged as a multi-member segmented tarball. The small target +// forces several members. +type segmentFixture struct { + entries []tarpack.Entry + index segmentIndex + data []byte } -// makeSegmentRepo creates n files of growing size under a repo directory and returns -// the repo path and its snapshot file listing. -func makeSegmentRepo(t *testing.T, n int) (string, []snapshotFile) { +func newSegmentFixture(t *testing.T) segmentFixture { t.Helper() repo := filepath.Join(t.TempDir(), "code") - for i := range n { + for i := range 30 { writeRepoFile(t, repo, fmt.Sprintf("pkg%d/file%d.py", i%3, i), strings.Repeat(fmt.Sprintf("x = %d\n", i), 50*i+1)) } files, err := snapshotFiles(t.Context(), repo, nil, false) require.NoError(t, err) - return repo, files + entries, sizes := plainTarEntries(repo, files) + + out := filepath.Join(t.TempDir(), "snap.tar.gz") + index, err := writeSegmentedTarball(t.Context(), "code", out, entries, sizes, 4096, 16) + require.NoError(t, err) + require.Greater(t, len(index.Members), 1) + data, err := os.ReadFile(out) + require.NoError(t, err) + return segmentFixture{entries: entries, index: index, data: data} +} + +func (f segmentFixture) names() []string { + var names []string + for _, e := range f.entries { + names = append(names, e.Name) + } + return names } func TestPlanSegments(t *testing.T) { @@ -104,129 +103,94 @@ func TestPlanSegments(t *testing.T) { // Old launchers read the whole .tar.gz as one gzip stream: it must yield exactly the // expected entries, even with several members. func TestSegmentedTarballReadsAsSingleStream(t *testing.T) { - ctx := t.Context() - repo, files := makeSegmentRepo(t, 30) - entries, sizes := plainTarEntries(repo, files) - out := filepath.Join(t.TempDir(), "snap.tar.gz") - index, err := writeSegmentedTarball(ctx, filepath.Base(repo), out, entries, sizes, 4096, 16) + f := newSegmentFixture(t) + gz, err := gzip.NewReader(bytes.NewReader(f.data)) require.NoError(t, err) - require.Greater(t, len(index.Members), 1) - - data, err := os.ReadFile(out) - require.NoError(t, err) - var want []string - for _, e := range entries { - want = append(want, e.Name) - } - assert.ElementsMatch(t, want, readSingleStream(t, data)) + assert.Equal(t, f.names(), tarNames(t, gz)) } -// Each member decodes independently to a valid tar fragment, and the union of the -// members is the full set of entries in order. +// Each member decodes on its own to a tar fragment, and the members in order are the full +// entry set. Only the last member carries the end-of-archive marker. func TestSegmentedTarballMembersDecodeIndependently(t *testing.T) { - ctx := t.Context() - repo, files := makeSegmentRepo(t, 30) - entries, sizes := plainTarEntries(repo, files) - out := filepath.Join(t.TempDir(), "snap.tar.gz") - index, err := writeSegmentedTarball(ctx, filepath.Base(repo), out, entries, sizes, 4096, 16) - require.NoError(t, err) - require.Greater(t, len(index.Members), 1) - - data, err := os.ReadFile(out) - require.NoError(t, err) + f := newSegmentFixture(t) var union []string - for _, m := range index.Members { - union = append(union, readMemberFragment(t, data[m.Offset:m.Offset+m.Length])...) - } - var want []string - for _, e := range entries { - want = append(want, e.Name) + for i, m := range f.index.Members { + member := f.data[m.Offset : m.Offset+m.Length] + gz, err := gzip.NewReader(bytes.NewReader(member)) + require.NoError(t, err) + payload, err := io.ReadAll(gz) + require.NoError(t, err) + endsWithMarker := bytes.HasSuffix(payload, make([]byte, 1024)) + assert.Equal(t, i == len(f.index.Members)-1, endsWithMarker, "member %d", i) + + gz, err = gzip.NewReader(bytes.NewReader(member)) + require.NoError(t, err) + union = append(union, tarNames(t, gz)...) } - assert.Equal(t, want, union) + assert.Equal(t, f.names(), union) } -// The index matches the tarball: members are contiguous, cover [0, size), size equals -// the file size, and every entry sits under the component directory. +// The index matches the tarball: members are contiguous, cover [0, size) with size equal +// to the file size, and every entry sits under the component directory. func TestSegmentedTarballIndexContract(t *testing.T) { - ctx := t.Context() - repo, files := makeSegmentRepo(t, 30) - entries, sizes := plainTarEntries(repo, files) - out := filepath.Join(t.TempDir(), "snap.tar.gz") - index, err := writeSegmentedTarball(ctx, filepath.Base(repo), out, entries, sizes, 4096, 16) - require.NoError(t, err) - - info, err := os.Stat(out) - require.NoError(t, err) - assert.Equal(t, segmentFormat, index.Format) - assert.Equal(t, segmentFormatVersion, index.Version) - assert.Equal(t, "code", index.Component) - assert.Equal(t, info.Size(), index.Size) + f := newSegmentFixture(t) + assert.Equal(t, segmentFormat, f.index.Format) + assert.Equal(t, segmentFormatVersion, f.index.Version) + assert.Equal(t, "code", f.index.Component) + assert.Equal(t, int64(len(f.data)), f.index.Size) var next int64 - for _, m := range index.Members { + for _, m := range f.index.Members { assert.Equal(t, next, m.Offset) assert.Positive(t, m.Length) next += m.Length } - assert.Equal(t, index.Size, next) + assert.Equal(t, f.index.Size, next) - data, err := os.ReadFile(out) - require.NoError(t, err) - for _, name := range readSingleStream(t, data) { + for _, name := range f.names() { assert.True(t, strings.HasPrefix(name, "code/"), name) } } -// Only the last member carries the end-of-archive marker. -func TestSegmentedTarballEndMarkerOnlyInLastMember(t *testing.T) { - ctx := t.Context() - repo, files := makeSegmentRepo(t, 30) - entries, sizes := plainTarEntries(repo, files) - out := filepath.Join(t.TempDir(), "snap.tar.gz") - index, err := writeSegmentedTarball(ctx, filepath.Base(repo), out, entries, sizes, 4096, 16) - require.NoError(t, err) - require.Greater(t, len(index.Members), 1) - - data, err := os.ReadFile(out) - require.NoError(t, err) - for i, m := range index.Members { - gz, err := gzip.NewReader(bytes.NewReader(data[m.Offset : m.Offset+m.Length])) - require.NoError(t, err) - payload, err := io.ReadAll(gz) - require.NoError(t, err) - trailer := make([]byte, 1024) - endsWithMarker := bytes.Equal(payload[len(payload)-1024:], trailer) - assert.Equal(t, i == len(index.Members)-1, endsWithMarker, "member %d", i) - } -} - -// A small repo packages as one member, and the tarball is a plain single-member archive. +// A small repo packages as one member, and the index covers the whole file. func TestCreatePlainTarballSmallRepoIsOneMember(t *testing.T) { - repo, files := makeSegmentRepo(t, 3) + repo := filepath.Join(t.TempDir(), "code") + writeRepoFile(t, repo, "a.py", "print()") + files, err := snapshotFiles(t.Context(), repo, nil, false) + require.NoError(t, err) out := filepath.Join(t.TempDir(), "snap.tar.gz") index, err := createPlainTarball(t.Context(), repo, out, files) require.NoError(t, err) - assert.Len(t, index.Members, 1) - assert.Equal(t, []segmentMember{{Offset: 0, Length: index.Size}}, index.Members) + + info, err := os.Stat(out) + require.NoError(t, err) + assert.Equal(t, []segmentMember{{Offset: 0, Length: info.Size()}}, index.Members) + assert.Equal(t, info.Size(), index.Size) } -// Same input yields byte-identical tarballs and indexes, which the content-addressed -// cache key depends on. +// Same input yields byte-identical tarballs and indexes, which the content-addressed cache +// key depends on. func TestCreatePlainTarballIsDeterministic(t *testing.T) { - repo, files := makeSegmentRepo(t, 30) - dir := t.TempDir() - first := filepath.Join(dir, "first.tar.gz") - second := filepath.Join(dir, "second.tar.gz") - entries, sizes := plainTarEntries(repo, files) - indexA, err := writeSegmentedTarball(t.Context(), filepath.Base(repo), first, entries, sizes, 4096, 16) - require.NoError(t, err) - indexB, err := writeSegmentedTarball(t.Context(), filepath.Base(repo), second, entries, sizes, 4096, 16) + repo := filepath.Join(t.TempDir(), "code") + for i := range 30 { + writeRepoFile(t, repo, fmt.Sprintf("f%d.py", i), strings.Repeat("x", 300*i+1)) + } + files, err := snapshotFiles(t.Context(), repo, nil, false) require.NoError(t, err) + entries, sizes := plainTarEntries(repo, files) - a, err := os.ReadFile(first) - require.NoError(t, err) - b, err := os.ReadFile(second) - require.NoError(t, err) - assert.Equal(t, a, b) + dir := t.TempDir() + write := func(name string) (segmentIndex, []byte) { + out := filepath.Join(dir, name) + index, err := writeSegmentedTarball(t.Context(), "code", out, entries, sizes, 4096, 16) + require.NoError(t, err) + data, err := os.ReadFile(out) + require.NoError(t, err) + return index, data + } + indexA, dataA := write("a.tar.gz") + indexB, dataB := write("b.tar.gz") + require.Greater(t, len(indexA.Members), 1) + assert.Equal(t, dataA, dataB) assert.Equal(t, indexA, indexB) } diff --git a/cmd/air/snapshot_upload.go b/cmd/air/snapshot_upload.go index 1af77c6d52d..b22362457ab 100644 --- a/cmd/air/snapshot_upload.go +++ b/cmd/air/snapshot_upload.go @@ -10,6 +10,7 @@ import ( "os" "path" "path/filepath" + "slices" "strings" "time" @@ -17,6 +18,7 @@ import ( "github.com/databricks/cli/libs/filer" "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go" + "golang.org/x/sync/errgroup" ) // uploadProvenanceSidecars controls collection and upload of git_state.json @@ -178,11 +180,11 @@ func uploadSnapshotTarball(ctx context.Context, w *databricks.WorkspaceClient, r // upload the bytes now or reuse an object already in the store. remote := path.Join(uploadPath, tarName) - exists, err := snapshotExists(ctx, f, tarName) + cached, err := snapshotCached(ctx, f, tarName, plan.mode == modePlainTar) if err != nil { return result, err } - if exists { + if cached { log.Debugf(ctx, "snapshot upload skipped; reusing %s", remote) result.CodeSourcePath = remote result.PackagingDurationMs = new(int64(0)) @@ -218,21 +220,50 @@ func uploadSnapshotTarball(ctx context.Context, w *databricks.WorkspaceClient, r } cmdio.LogProgress(ctx, fmt.Sprintf("Uploading %s...", tarName)) uploadStart := time.Now() - // The sidecar goes first so a tarball is never visible without its index. + // The sidecar and tarball are written concurrently. The job is only submitted once both + // succeed, and a later cache hit requires both (see snapshotCached). + g, gctx := errgroup.WithContext(ctx) if index != nil { - if err := uploadSegmentIndex(ctx, f, tarName, index); err != nil { - return result, err - } + g.Go(func() error { return uploadSegmentIndex(gctx, f, tarName, index) }) } - err = f.Write(ctx, tarName, file, filer.OverwriteIfExists, filer.CreateParentDirectories) + g.Go(func() error { + if err := f.Write(gctx, tarName, file, filer.OverwriteIfExists, filer.CreateParentDirectories); err != nil { + return fmt.Errorf("failed to upload snapshot %s: %w", tarName, err) + } + return nil + }) + err = g.Wait() result.UploadDurationMs = new(time.Since(uploadStart).Milliseconds()) if err != nil { - return result, fmt.Errorf("failed to upload snapshot %s: %w", tarName, err) + return result, err } result.CodeSourcePath = remote return result, nil } +// snapshotCached reports whether the snapshot is already in the store. For an indexed +// (plain_tar) snapshot both the tarball and its index sidecar must exist: a half-written +// pair from an interrupted upload is a miss, and the pair is re-uploaded. +func snapshotCached(ctx context.Context, store filer.Filer, tarName string, indexed bool) (bool, error) { + names := []string{tarName} + if indexed { + names = append(names, tarName+segmentIndexSuffix) + } + exists := make([]bool, len(names)) + g, gctx := errgroup.WithContext(ctx) + for i, name := range names { + g.Go(func() error { + ok, err := snapshotExists(gctx, store, name) + exists[i] = ok + return err + }) + } + if err := g.Wait(); err != nil { + return false, err + } + return !slices.Contains(exists, false), nil +} + // uploadSegmentIndex writes the index sidecar for tarName next to it in the store. func uploadSegmentIndex(ctx context.Context, store filer.Filer, tarName string, index *segmentIndex) error { data, err := json.Marshal(index) diff --git a/cmd/air/snapshot_upload_test.go b/cmd/air/snapshot_upload_test.go new file mode 100644 index 00000000000..3c6c7af64f8 --- /dev/null +++ b/cmd/air/snapshot_upload_test.go @@ -0,0 +1,143 @@ +package aircmd + +import ( + "encoding/json" + "path" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/filer" + "github.com/databricks/cli/libs/testserver" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// uploadRecorder records which snapshot objects the fake workspace receives. The import +// handler runs concurrently with the sidecar and tarball writes, so it is mutex-guarded. +type uploadRecorder struct { + mu sync.Mutex + written map[string]bool + index segmentIndex +} + +// reset forgets prior writes so a phase can assert on its own uploads. +func (r *uploadRecorder) reset() { + r.mu.Lock() + defer r.mu.Unlock() + r.written = map[string]bool{} +} + +// names returns the snapshot object names written since the last reset. +func (r *uploadRecorder) names() map[string]bool { + r.mu.Lock() + defer r.mu.Unlock() + out := map[string]bool{} + for k := range r.written { + out[k] = true + } + return out +} + +func (r *uploadRecorder) record(name string, body []byte) { + r.mu.Lock() + defer r.mu.Unlock() + r.written[name] = true + if strings.HasSuffix(name, segmentIndexSuffix) { + _ = json.Unmarshal(body, &r.index) + } +} + +// newUploadTestWorkspace returns a fake workspace that records snapshot object writes. +func newUploadTestWorkspace(t *testing.T) (*databricks.WorkspaceClient, *uploadRecorder) { + t.Helper() + server := testserver.New(t) + t.Cleanup(server.Close) + server.Handle("POST", "/api/2.2/jobs/runs/submit", func(req testserver.Request) any { + return jobs.SubmitRunResponse{RunId: 555} + }) + rec := &uploadRecorder{written: map[string]bool{}} + server.Handle("POST", "/api/2.0/workspace-files/import-file/{path...}", func(req testserver.Request) any { + p := req.Vars["path"] + if strings.Contains(p, "/.air/repo_snapshots/") { + rec.record(path.Base(p), req.Body) + } + return req.Workspace.WorkspaceFilesImportFile(p, req.Body, req.URL.Query().Get("overwrite") == "true") + }) + stubValidateConfig(server) + testserver.AddDefaultHandlers(server) + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "token"}) + require.NoError(t, err) + return w, rec +} + +// A plain-tar snapshot writes its index sidecar and tarball, and a later submit of the +// same tree is a cache hit that writes nothing. +func TestUploadSnapshotPlainTarWritesIndexAndTarball(t *testing.T) { + w, rec := newUploadTestWorkspace(t) + repo := filepath.Join(t.TempDir(), "src") + writeRepoFile(t, repo, "train.py", "print()") + snap, cfgPath := loadSnapshotConfig(t, repo) + + ctx := cmdio.MockDiscard(t.Context()) + sidecarStore, sidecarBase := testSidecarStore(t, w) + first, err := uploadSnapshot(ctx, w, snap, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase) + require.NoError(t, err) + + tarName := path.Base(first.CodeSourcePath) + assert.Equal(t, map[string]bool{tarName: true, tarName + segmentIndexSuffix: true}, rec.names()) + assert.Equal(t, segmentFormat, rec.index.Format) + assert.Equal(t, "src", rec.index.Component) + require.NotNil(t, first.SizeBytes) + assert.Equal(t, *first.SizeBytes, rec.index.Size) + assert.Len(t, rec.index.Members, 1) + + rec.reset() + second, err := uploadSnapshot(ctx, w, snap, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase) + require.NoError(t, err) + assert.Equal(t, first.CodeSourcePath, second.CodeSourcePath) + assert.Empty(t, rec.names(), "cache hit should write neither the index nor the tarball") +} + +// A tarball without its index sidecar is a cache miss: both objects are written again. +func TestUploadSnapshotPlainTarMissingSidecarReuploadsBoth(t *testing.T) { + w, rec := newUploadTestWorkspace(t) + repo := filepath.Join(t.TempDir(), "src") + writeRepoFile(t, repo, "train.py", "print()") + snap, cfgPath := loadSnapshotConfig(t, repo) + + ctx := cmdio.MockDiscard(t.Context()) + sidecarStore, sidecarBase := testSidecarStore(t, w) + first, err := uploadSnapshot(ctx, w, snap, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase) + require.NoError(t, err) + tarName := path.Base(first.CodeSourcePath) + + uploadDir, err := filer.NewWorkspaceFilesClient(w, testSnapshotArtifactPath+"/.internal") + require.NoError(t, err) + require.NoError(t, uploadDir.Delete(ctx, tarName+segmentIndexSuffix)) + + rec.reset() + second, err := uploadSnapshot(ctx, w, snap, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase) + require.NoError(t, err) + assert.Equal(t, first.CodeSourcePath, second.CodeSourcePath) + assert.Equal(t, map[string]bool{tarName: true, tarName + segmentIndexSuffix: true}, rec.names()) +} + +// loadSnapshotConfig writes a run config whose code_source is repo and returns its snapshot +// config and path. +func loadSnapshotConfig(t *testing.T, repo string) (*snapshotSourceConfig, string) { + t.Helper() + cfgPath := writeConfigFile(t, "run.yaml", minimalConfig+` +code_source: + type: snapshot + snapshot: + root_path: `+repo+` +`) + loaded, err := loadRunConfig(cfgPath) + require.NoError(t, err) + return loaded.CodeSource.Snapshot, cfgPath +} From bd1809c4ea5d44d5849eb2ca9f71f6e24c04faf2 Mon Sep 17 00:00:00 2001 From: Steven Chen Date: Mon, 9 Nov 2026 08:24:41 +0000 Subject: [PATCH 3/4] Add changelog fragment for segmented snapshots Co-authored-by: Isaac --- .nextchanges/air/segmented-snapshot.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 .nextchanges/air/segmented-snapshot.md diff --git a/.nextchanges/air/segmented-snapshot.md b/.nextchanges/air/segmented-snapshot.md new file mode 100644 index 00000000000..32ad18367ac --- /dev/null +++ b/.nextchanges/air/segmented-snapshot.md @@ -0,0 +1 @@ +* `databricks air run` packages working-tree code snapshots as parallel gzip segments with an index, which shortens submission for large code directories. ([#7403](https://github.com/databricks/cli/pull/7403)) From 07458fb9c5e1b220c6fab84d0cebd6ff3a870a86 Mon Sep 17 00:00:00 2001 From: Steven Chen Date: Mon, 9 Nov 2026 08:44:51 +0000 Subject: [PATCH 4/4] air: spool snapshot members to temp files with bounded concurrency Members are compressed into temp files, at most runtime.NumCPU() at a time, and concatenated only after every member succeeds, so peak memory no longer scales with the tarball size. Members use compress/gzip: pgzip's per-writer pipelines took about 1 GB peak RSS on the 516 MB benchmark repo, against 78 MB with compress/gzip. Co-authored-by: Isaac --- cmd/air/snapshot_segment.go | 83 +++++++++++++++++++++++++------------ 1 file changed, 57 insertions(+), 26 deletions(-) diff --git a/cmd/air/snapshot_segment.go b/cmd/air/snapshot_segment.go index 97f61c5fb50..6710fb6a6fa 100644 --- a/cmd/air/snapshot_segment.go +++ b/cmd/air/snapshot_segment.go @@ -1,14 +1,15 @@ package aircmd import ( - "bytes" + "compress/gzip" "context" "fmt" "io" "os" + "path/filepath" + "runtime" "github.com/databricks/cli/libs/tarpack" - "github.com/klauspost/pgzip" "golang.org/x/sync/errgroup" ) @@ -99,45 +100,65 @@ func writeSegmentedTarball(ctx context.Context, component, outputTarball string, }) } -// writeSegments runs writeMember for n members concurrently, each into its own buffer, and -// appends the buffers to outputTarball in member order as they complete. Compressed -// members are held in memory until their turn, so peak memory is about the compressed -// size of the tarball, and no copy goes through a temp file. +// writeSegments runs writeMember for n members, spooling each to a temp file with at most +// runtime.NumCPU() in flight, then appends the files to outputTarball in member order. The +// output is assembled only after every member succeeds, so a failed member never leaves a +// partial tarball behind. func writeSegments(ctx context.Context, component, outputTarball string, n int, writeMember func(ctx context.Context, i int, w io.Writer) error) (segmentIndex, error) { - out, err := os.Create(outputTarball) + dir, err := os.MkdirTemp("", "air-snapshot-members-") if err != nil { - return segmentIndex{}, fmt.Errorf("failed to create tarball: %w", err) + return segmentIndex{}, fmt.Errorf("failed to create snapshot members dir: %w", err) } - defer out.Close() + defer os.RemoveAll(dir) - ctx, cancel := context.WithCancel(ctx) - defer cancel() - bufs := make([]*bytes.Buffer, n) - done := make([]chan struct{}, n) + paths := make([]string, n) g, gctx := errgroup.WithContext(ctx) + g.SetLimit(runtime.NumCPU()) for i := range n { - bufs[i] = &bytes.Buffer{} - done[i] = make(chan struct{}) + paths[i] = filepath.Join(dir, fmt.Sprintf("member-%d", i)) g.Go(func() error { - defer close(done[i]) - return writeMember(gctx, i, bufs[i]) + return spoolFile(gctx, paths[i], func(ctx context.Context, w io.Writer) error { + return writeMember(ctx, i, w) + }) }) } + if err := g.Wait(); err != nil { + return segmentIndex{}, err + } + return concatMembers(component, outputTarball, paths) +} + +// spoolFile creates path and fills it with the output of write. +func spoolFile(ctx context.Context, path string, write func(ctx context.Context, w io.Writer) error) error { + f, err := os.Create(path) + if err != nil { + return err + } + if err := write(ctx, f); err != nil { + f.Close() + return err + } + return f.Close() +} + +// concatMembers appends the member files at paths, in order, to outputTarball and returns +// the index of their byte ranges. +func concatMembers(component, outputTarball string, paths []string) (segmentIndex, error) { + out, err := os.Create(outputTarball) + if err != nil { + return segmentIndex{}, fmt.Errorf("failed to create tarball: %w", err) + } + defer out.Close() index := segmentIndex{Format: segmentFormat, Version: segmentFormatVersion, Component: component} - for i := range n { - <-done[i] - length, err := bufs[i].WriteTo(out) + for _, p := range paths { + length, err := appendFile(out, p) if err != nil { - return segmentIndex{}, fmt.Errorf("failed to write snapshot member %d: %w", i, err) + return segmentIndex{}, fmt.Errorf("failed to append snapshot member: %w", err) } - bufs[i] = nil index.Members = append(index.Members, segmentMember{Offset: index.Size, Length: length}) index.Size += length } - if err := g.Wait(); err != nil { - return segmentIndex{}, err - } if err := out.Close(); err != nil { return segmentIndex{}, fmt.Errorf("failed to finalize tarball: %w", err) } @@ -147,7 +168,7 @@ func writeSegments(ctx context.Context, component, outputTarball string, n int, // writeMemberTar writes entries to w as one gzip member. Members other than the last // omit the tar end-of-archive marker. func writeMemberTar(ctx context.Context, w io.Writer, entries []tarpack.Entry, last bool) error { - gz, err := pgzip.NewWriterLevel(ctxWriter{ctx, w}, pgzip.DefaultCompression) + gz, err := gzip.NewWriterLevel(ctxWriter{ctx, w}, gzip.DefaultCompression) if err != nil { return err } @@ -163,3 +184,13 @@ func writeMemberTar(ctx context.Context, w io.Writer, entries []tarpack.Entry, l } return nil } + +// appendFile copies the file at path onto the end of w and returns the bytes copied. +func appendFile(w io.Writer, path string) (int64, error) { + f, err := os.Open(path) + if err != nil { + return 0, err + } + defer f.Close() + return io.Copy(w, f) +}