From dd298a456e41231aa84c12283eae80a5898671d8 Mon Sep 17 00:00:00 2001 From: Steven Chen Date: Mon, 9 Nov 2026 07:18:28 +0000 Subject: [PATCH 1/3] air: package git-pinned snapshots with parallel gzip createGitArchiveSnapshot ran `git archive --format=tar.gz`, which compresses on a single thread. Stream `git archive --format=tar` into a klauspost/pgzip writer instead, matching the plain_tar path. The git runner gains runTo, which streams stdout to a writer; runBytes delegates to it. On a 516 MB / 15k-file repo packaging drops from 12.2 s to 1.1 s. Co-authored-by: Isaac --- cmd/air/snapshot_git.go | 21 ++++++-- cmd/air/snapshot_package.go | 31 ++++++++--- cmd/air/snapshot_package_test.go | 92 ++++++++++++++++++++++++++++++++ 3 files changed, 132 insertions(+), 12 deletions(-) diff --git a/cmd/air/snapshot_git.go b/cmd/air/snapshot_git.go index 6172ccec9c6..69972dd4d2b 100644 --- a/cmd/air/snapshot_git.go +++ b/cmd/air/snapshot_git.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "os/exec" "strings" "time" @@ -35,20 +36,30 @@ func (g gitRepo) run(ctx context.Context, args ...string) (string, error) { // runBytes is run returning raw stdout bytes, for the dirty-diff capture which needs // exact bytes and a size measurement. func (g gitRepo) runBytes(ctx context.Context, args ...string) ([]byte, error) { + var stdout bytes.Buffer + if err := g.runTo(ctx, &stdout, args...); err != nil { + return nil, err + } + return stdout.Bytes(), nil +} + +// runTo is run streaming stdout into w as git produces it, for output too large to +// buffer such as `git archive`. +func (g gitRepo) runTo(ctx context.Context, w io.Writer, args ...string) error { full := append([]string{"-C", g.path}, args...) cmd := exec.CommandContext(ctx, "git", full...) - var stdout, stderr bytes.Buffer - cmd.Stdout = &stdout + var stderr bytes.Buffer + cmd.Stdout = w cmd.Stderr = &stderr if err := cmd.Run(); err != nil { msg := strings.TrimSpace(stderr.String()) if msg == "" { - return nil, fmt.Errorf("git %s: %w", strings.Join(args, " "), err) + return fmt.Errorf("git %s: %w", strings.Join(args, " "), err) } - return nil, fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, msg) + return fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, msg) } - return stdout.Bytes(), nil + return nil } // isRepository reports whether the path is inside a git work tree. Using diff --git a/cmd/air/snapshot_package.go b/cmd/air/snapshot_package.go index ca8a6942f5f..ce5c2b68825 100644 --- a/cmd/air/snapshot_package.go +++ b/cmd/air/snapshot_package.go @@ -21,9 +21,11 @@ import ( // load-bearing — the remote entry_script extracts to /databricks/code_source/ — so // both forms prefix every entry with it. -// createGitArchiveSnapshot writes a gzipped tar of commitSHA to outputTarball via -// `git archive`, with every entry prefixed by directoryName/. When subtreePrefix is -// set, only that repository subtree is archived and includePaths are relative to it. +// createGitArchiveSnapshot writes a gzipped tar of commitSHA to outputTarball, with every +// entry prefixed by directoryName/. The tar is streamed from `git archive --format=tar` +// into a parallel pgzip writer: git's own tar.gz format compresses on one thread, which +// dominates packaging time on a large repo. When subtreePrefix is set, only that +// repository subtree is archived and includePaths are relative to it. func createGitArchiveSnapshot(ctx context.Context, git gitRepo, commitSHA, outputTarball, directoryName string, includePaths []string, subtreePrefix string) error { treeish := commitSHA archiveGit := git @@ -38,16 +40,31 @@ func createGitArchiveSnapshot(ctx context.Context, git gitRepo, commitSHA, outpu args := []string{ "archive", - "--format=tar.gz", + "--format=tar", "--prefix=" + directoryName + "/", - "-o", outputTarball, treeish, } args = append(args, includePaths...) - if _, err := archiveGit.run(ctx, args...); err != nil { + + out, err := os.Create(outputTarball) + if err != nil { + return fmt.Errorf("failed to create tarball: %w", err) + } + gz, err := pgzip.NewWriterLevel(out, pgzip.DefaultCompression) + if err != nil { + out.Close() + return err + } + if err := archiveGit.runTo(ctx, ctxWriter{ctx, gz}, args...); err != nil { + gz.Close() + out.Close() return fmt.Errorf("failed to create git archive: %w", err) } - return nil + if err := gz.Close(); err != nil { + out.Close() + return fmt.Errorf("failed to finalize gzip: %w", err) + } + return out.Close() } // createPlainTarball writes a gzipped tar of the given working-tree files to diff --git a/cmd/air/snapshot_package_test.go b/cmd/air/snapshot_package_test.go index db42f634356..d3aad30165f 100644 --- a/cmd/air/snapshot_package_test.go +++ b/cmd/air/snapshot_package_test.go @@ -4,6 +4,7 @@ import ( "archive/tar" "compress/gzip" "context" + "io" "os" "path/filepath" "slices" @@ -123,6 +124,97 @@ func TestCreateGitArchiveSnapshot_SubdirectoryRootPathWithIncludePaths(t *testin assert.NotContains(t, entries, "subpkg/configs/train.yaml") } +// tarballFileContents returns the regular-file entries of a .tar.gz keyed by name. +func tarballFileContents(t *testing.T, path string) map[string]string { + t.Helper() + f, err := os.Open(path) + require.NoError(t, err) + defer f.Close() + + gz, err := gzip.NewReader(f) + require.NoError(t, err) + defer gz.Close() + + files := map[string]string{} + tr := tar.NewReader(gz) + for { + hdr, err := tr.Next() + if err != nil { + break + } + if hdr.Typeflag != tar.TypeReg { + continue + } + body, err := io.ReadAll(tr) + require.NoError(t, err) + files[hdr.Name] = string(body) + } + return files +} + +func TestCreateGitArchiveSnapshot_ExtractsExpectedTree(t *testing.T) { + ctx := t.Context() + repo := newTestRepo(t) + writeRepoFile(t, repo, "README.md", "repo root") + writeRepoFile(t, repo, "subpkg/train.py", "train") + writeRepoFile(t, repo, "subpkg/src/model.py", "model") + writeRepoFile(t, repo, "subpkg/configs/train.yaml", "cfg") + sha := commitAll(t, repo, "init") + + prefix, err := newGitRepo(filepath.Join(repo, "subpkg")).repoRelativePrefix(ctx) + require.NoError(t, err) + + tests := []struct { + name string + gitRepoPath string + dirName string + includePaths []string + subtreePrefix string + want map[string]string + }{ + { + name: "root", + gitRepoPath: repo, + dirName: "repo", + want: map[string]string{ + "repo/README.md": "repo root", + "repo/subpkg/train.py": "train", + "repo/subpkg/src/model.py": "model", + "repo/subpkg/configs/train.yaml": "cfg", + }, + }, + { + name: "subtree_with_include_paths", + gitRepoPath: filepath.Join(repo, "subpkg"), + dirName: "subpkg", + includePaths: []string{"src"}, + subtreePrefix: prefix, + want: map[string]string{ + "subpkg/src/model.py": "model", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + out := filepath.Join(t.TempDir(), "snap.tar.gz") + require.NoError(t, createGitArchiveSnapshot(ctx, newGitRepo(tt.gitRepoPath), sha, out, tt.dirName, tt.includePaths, tt.subtreePrefix)) + assert.Equal(t, tt.want, tarballFileContents(t, out)) + }) + } +} + +func TestCreateGitArchiveSnapshot_GitFailureSurfacesStderr(t *testing.T) { + ctx := t.Context() + repo := newTestRepo(t) + writeRepoFile(t, repo, "a.txt", "1") + commitAll(t, repo, "init") + + out := filepath.Join(t.TempDir(), "snap.tar.gz") + err := createGitArchiveSnapshot(ctx, newGitRepo(repo), "0000000000000000000000000000000000000000", out, "repo", nil, "") + require.ErrorContains(t, err, "failed to create git archive") + assert.ErrorContains(t, err, "fatal") +} + func TestCreatePlainTarball(t *testing.T) { ctx := t.Context() repo := t.TempDir() From f7bdd23718c1f6d0857dbfa632bc5ca40950f6a2 Mon Sep 17 00:00:00 2001 From: Steven Chen Date: Mon, 9 Nov 2026 08:17:08 +0000 Subject: [PATCH 2/3] air: address review on git archive packaging tests Drop the duplicate tree-content test (existing root/include/subtree tests cover the path). Add a cancelled-context test for the git path, matching the plain tarball one. Add the air changelog fragment. Co-authored-by: Isaac --- .nextchanges/air/git-archive-pgzip.md | 1 + cmd/air/snapshot_package_test.go | 91 ++++----------------------- 2 files changed, 12 insertions(+), 80 deletions(-) create mode 100644 .nextchanges/air/git-archive-pgzip.md diff --git a/.nextchanges/air/git-archive-pgzip.md b/.nextchanges/air/git-archive-pgzip.md new file mode 100644 index 00000000000..f4a9820e728 --- /dev/null +++ b/.nextchanges/air/git-archive-pgzip.md @@ -0,0 +1 @@ +* Speed up `databricks air run` for git-pinned `code_source` snapshots by compressing the archive in parallel, which substantially shortens packaging on large repositories. ([#7402](https://github.com/databricks/cli/pull/7402)) diff --git a/cmd/air/snapshot_package_test.go b/cmd/air/snapshot_package_test.go index d3aad30165f..18705798cab 100644 --- a/cmd/air/snapshot_package_test.go +++ b/cmd/air/snapshot_package_test.go @@ -4,7 +4,6 @@ import ( "archive/tar" "compress/gzip" "context" - "io" "os" "path/filepath" "slices" @@ -124,85 +123,6 @@ func TestCreateGitArchiveSnapshot_SubdirectoryRootPathWithIncludePaths(t *testin assert.NotContains(t, entries, "subpkg/configs/train.yaml") } -// tarballFileContents returns the regular-file entries of a .tar.gz keyed by name. -func tarballFileContents(t *testing.T, path string) map[string]string { - t.Helper() - f, err := os.Open(path) - require.NoError(t, err) - defer f.Close() - - gz, err := gzip.NewReader(f) - require.NoError(t, err) - defer gz.Close() - - files := map[string]string{} - tr := tar.NewReader(gz) - for { - hdr, err := tr.Next() - if err != nil { - break - } - if hdr.Typeflag != tar.TypeReg { - continue - } - body, err := io.ReadAll(tr) - require.NoError(t, err) - files[hdr.Name] = string(body) - } - return files -} - -func TestCreateGitArchiveSnapshot_ExtractsExpectedTree(t *testing.T) { - ctx := t.Context() - repo := newTestRepo(t) - writeRepoFile(t, repo, "README.md", "repo root") - writeRepoFile(t, repo, "subpkg/train.py", "train") - writeRepoFile(t, repo, "subpkg/src/model.py", "model") - writeRepoFile(t, repo, "subpkg/configs/train.yaml", "cfg") - sha := commitAll(t, repo, "init") - - prefix, err := newGitRepo(filepath.Join(repo, "subpkg")).repoRelativePrefix(ctx) - require.NoError(t, err) - - tests := []struct { - name string - gitRepoPath string - dirName string - includePaths []string - subtreePrefix string - want map[string]string - }{ - { - name: "root", - gitRepoPath: repo, - dirName: "repo", - want: map[string]string{ - "repo/README.md": "repo root", - "repo/subpkg/train.py": "train", - "repo/subpkg/src/model.py": "model", - "repo/subpkg/configs/train.yaml": "cfg", - }, - }, - { - name: "subtree_with_include_paths", - gitRepoPath: filepath.Join(repo, "subpkg"), - dirName: "subpkg", - includePaths: []string{"src"}, - subtreePrefix: prefix, - want: map[string]string{ - "subpkg/src/model.py": "model", - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - out := filepath.Join(t.TempDir(), "snap.tar.gz") - require.NoError(t, createGitArchiveSnapshot(ctx, newGitRepo(tt.gitRepoPath), sha, out, tt.dirName, tt.includePaths, tt.subtreePrefix)) - assert.Equal(t, tt.want, tarballFileContents(t, out)) - }) - } -} - func TestCreateGitArchiveSnapshot_GitFailureSurfacesStderr(t *testing.T) { ctx := t.Context() repo := newTestRepo(t) @@ -215,6 +135,17 @@ func TestCreateGitArchiveSnapshot_GitFailureSurfacesStderr(t *testing.T) { assert.ErrorContains(t, err, "fatal") } +func TestCreateGitArchiveSnapshotStopsWhenCancelled(t *testing.T) { + repo := newTestRepo(t) + writeRepoFile(t, repo, "a.txt", "1") + sha := commitAll(t, repo, "init") + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + err := createGitArchiveSnapshot(ctx, newGitRepo(repo), sha, filepath.Join(t.TempDir(), "snap.tar.gz"), "repo", nil, "") + require.ErrorIs(t, err, context.Canceled) +} + func TestCreatePlainTarball(t *testing.T) { ctx := t.Context() repo := t.TempDir() From 6c67d334075f2b9043939ea8e3f69283d151ca0c Mon Sep 17 00:00:00 2001 From: Steven Chen Date: Mon, 9 Nov 2026 08:53:41 +0000 Subject: [PATCH 3/3] air: size git_archive members from the blob total and spool them Member sizes come from `git ls-tree` blob sizes for the same tree, with the target max(32 MiB, ceil(total/16)), so the members stay balanced for archives of any size. The archive is streamed into raw per-member temp files, which are compressed concurrently (at most NumCPU at a time) and concatenated after all succeed. Spooling raw tar first keeps the stream from waiting on one compressor at a time. The packaging version is bumped to v2 (and the golden keys regenerated), which also re-keys plain_tar: one extra re-upload after upgrading. Co-authored-by: Isaac --- .nextchanges/air/git-segmented-snapshot.md | 1 + cmd/air/runsubmit_test.go | 5 +- cmd/air/snapshot_cachekey.go | 2 +- cmd/air/snapshot_git_segment_test.go | 105 +++++++++++++++ cmd/air/snapshot_package.go | 97 +++++++++++--- cmd/air/snapshot_package_test.go | 12 +- cmd/air/snapshot_segment.go | 143 +++++++++++++++++++++ cmd/air/snapshot_upload.go | 36 ++---- cmd/air/testdata/cache_keys.json | 18 +-- 9 files changed, 358 insertions(+), 61 deletions(-) create mode 100644 .nextchanges/air/git-segmented-snapshot.md create mode 100644 cmd/air/snapshot_git_segment_test.go diff --git a/.nextchanges/air/git-segmented-snapshot.md b/.nextchanges/air/git-segmented-snapshot.md new file mode 100644 index 00000000000..af2a7b255a8 --- /dev/null +++ b/.nextchanges/air/git-segmented-snapshot.md @@ -0,0 +1 @@ +* `databricks air run` with a pinned git commit packages the snapshot as parallel gzip segments with an index, which shortens submission for large repositories. ([#7404](https://github.com/databricks/cli/pull/7404)) diff --git a/cmd/air/runsubmit_test.go b/cmd/air/runsubmit_test.go index 98bd7f9a4e0..e6eb46e4491 100644 --- a/cmd/air/runsubmit_test.go +++ b/cmd/air/runsubmit_test.go @@ -904,11 +904,14 @@ func TestSubmitWorkloadGitArchiveCaching(t *testing.T) { // persistence so the second submit's cache-existence Stat sees the first upload. // Dedupe by path: the filer mkdirs-and-retries the import on a missing // parent dir, so one logical upload can hit this route more than once. + var mu sync.Mutex uploaded := map[string]bool{} server.Handle("POST", "/api/2.0/workspace-files/import-file/{path...}", func(req testserver.Request) any { p := req.Vars["path"] if strings.Contains(p, "/.air/repo_snapshots/") { + mu.Lock() uploaded[p] = true + mu.Unlock() } return req.Workspace.WorkspaceFilesImportFile(p, req.Body, req.URL.Query().Get("overwrite") == "true") }) @@ -943,7 +946,7 @@ code_source: // Same pinned commit → identical content-addressed remote path, uploaded once // (the second submit is a cache hit and moves no bytes). assert.Equal(t, first.CodeSourcePath, second.CodeSourcePath) - assert.Len(t, uploaded, 1, "git_archive cache hit should skip the second upload") + assert.Len(t, uploaded, 2, "git_archive cache hit should skip the second upload: tarball and index sidecar") require.NotNil(t, first.SizeBytes) assert.Positive(t, *first.SizeBytes) assert.Nil(t, second.SizeBytes) diff --git a/cmd/air/snapshot_cachekey.go b/cmd/air/snapshot_cachekey.go index 7b278a3a8cf..b130b17b56d 100644 --- a/cmd/air/snapshot_cachekey.go +++ b/cmd/air/snapshot_cachekey.go @@ -14,7 +14,7 @@ import ( ) // snapshotPackagingVersion is bumped when packaging logic changes in a way that invalidates existing caches -const snapshotPackagingVersion = "v1" +const snapshotPackagingVersion = "v2" // plainTarKeyVersion namespaces the plain_tar working-tree key (so it can never collide // with a git_archive key) and lets us invalidate it if the fingerprint scheme changes. diff --git a/cmd/air/snapshot_git_segment_test.go b/cmd/air/snapshot_git_segment_test.go new file mode 100644 index 00000000000..e6955eeab8f --- /dev/null +++ b/cmd/air/snapshot_git_segment_test.go @@ -0,0 +1,105 @@ +package aircmd + +import ( + "bytes" + "compress/gzip" + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// mustCreateGitArchiveSnapshot packages commitSHA with createGitArchiveSnapshot and fails the +// test on error. +func mustCreateGitArchiveSnapshot(t *testing.T, ctx context.Context, git gitRepo, commitSHA, outputTarball, directoryName string, includePaths []string, subtreePrefix string) { + t.Helper() + _, err := createGitArchiveSnapshot(ctx, git, commitSHA, outputTarball, directoryName, includePaths, subtreePrefix) + require.NoError(t, err) +} + +// commitFiles commits n files of size bytes each, spread over three directories. +func commitFiles(t *testing.T, n, size int) (string, string) { + t.Helper() + repo := newTestRepo(t) + for i := range n { + writeRepoFile(t, repo, fmt.Sprintf("pkg%d/file%d.py", i%3, i), strings.Repeat("x", size+i)) + } + return repo, commitAll(t, repo, "init") +} + +// A multi-member git_archive snapshot reads as one stream with pax_global_header first. Its +// members are contiguous, decode independently to the same entries, and cover the file. +func TestCreateGitArchiveSnapshotSegmented(t *testing.T) { + repo, sha := commitFiles(t, 30, 200) + dirName := filepath.Base(repo) + + out := filepath.Join(t.TempDir(), "snap.tar.gz") + index, err := createGitArchiveSegments(t.Context(), newGitRepo(repo), sha, out, dirName, nil, "", 4096) + require.NoError(t, err) + require.Greater(t, len(index.Members), 1) + assert.Equal(t, dirName, index.Component) + + data, err := os.ReadFile(out) + require.NoError(t, err) + assert.Equal(t, int64(len(data)), index.Size) + + gz, err := gzip.NewReader(bytes.NewReader(data)) + require.NoError(t, err) + single := tarNames(t, gz) + require.NotEmpty(t, single) + assert.Equal(t, "pax_global_header", single[0]) + + var union []string + var next int64 + for i, m := range index.Members { + assert.Equal(t, next, m.Offset, "member %d not contiguous", i) + next += m.Length + member, err := gzip.NewReader(bytes.NewReader(data[m.Offset : m.Offset+m.Length])) + require.NoError(t, err) + names := tarNames(t, member) + if i == 0 { + require.NotEmpty(t, names) + assert.Equal(t, "pax_global_header", names[0]) + } + union = append(union, names...) + } + assert.Equal(t, index.Size, next) + assert.Equal(t, single, union) +} + +// An archive larger than maxMembers times the minimum target still splits into balanced +// members, instead of the last member absorbing the rest. Balance is checked on uncompressed +// tar bytes, which is what the launcher parallelizes over. +func TestCreateGitArchiveSnapshotBalancesLargeArchive(t *testing.T) { + const minTarget = 4096 + // 64 files of about 4 KiB: about 256 KiB of blobs, well over 16 times the 4 KiB minimum. + repo, sha := commitFiles(t, 64, 4000) + out := filepath.Join(t.TempDir(), "snap.tar.gz") + index, err := createGitArchiveSegments(t.Context(), newGitRepo(repo), sha, out, filepath.Base(repo), nil, "", minTarget) + require.NoError(t, err) + require.LessOrEqual(t, len(index.Members), segmentMaxMembers) + require.Greater(t, len(index.Members), 1) + + data, err := os.ReadFile(out) + require.NoError(t, err) + var sizes []int64 + var total int64 + for _, m := range index.Members { + gz, err := gzip.NewReader(bytes.NewReader(data[m.Offset : m.Offset+m.Length])) + require.NoError(t, err) + n, err := io.Copy(io.Discard, gz) + require.NoError(t, err) + sizes = append(sizes, n) + total += n + } + average := total / int64(len(sizes)) + for i, n := range sizes { + assert.LessOrEqual(t, n, 2*average, "member %d holds %d of %d uncompressed bytes", i, n, total) + } +} diff --git a/cmd/air/snapshot_package.go b/cmd/air/snapshot_package.go index 3073b36b7fb..c86ddbcc58d 100644 --- a/cmd/air/snapshot_package.go +++ b/cmd/air/snapshot_package.go @@ -9,6 +9,7 @@ import ( "os" "os/exec" "path/filepath" + "strconv" "strings" "github.com/databricks/cli/libs/tarpack" @@ -20,23 +21,36 @@ import ( // load-bearing — the remote entry_script extracts to /databricks/code_source/ — so // both forms prefix every entry with it. -// createGitArchiveSnapshot writes a gzipped tar of commitSHA to outputTarball, with every -// entry prefixed by directoryName/. The tar is streamed from `git archive --format=tar` -// into a parallel pgzip writer: git's own tar.gz format compresses on one thread, which -// dominates packaging time on a large repo. When subtreePrefix is set, only that -// repository subtree is archived and includePaths are relative to it. -func createGitArchiveSnapshot(ctx context.Context, git gitRepo, commitSHA, outputTarball, directoryName string, includePaths []string, subtreePrefix string) error { +// createGitArchiveSnapshot writes commitSHA to outputTarball as a segmented tarball (see +// snapshot_segment.go) and returns its index. Every entry is prefixed by directoryName/. +// The tar is streamed from `git archive --format=tar`, since git's own tar.gz format +// compresses on one thread. When subtreePrefix is set, only that repository subtree is +// archived and includePaths are relative to it. +func createGitArchiveSnapshot(ctx context.Context, git gitRepo, commitSHA, outputTarball, directoryName string, includePaths []string, subtreePrefix string) (segmentIndex, error) { + return createGitArchiveSegments(ctx, git, commitSHA, outputTarball, directoryName, includePaths, subtreePrefix, segmentTargetBytes) +} + +// createGitArchiveSegments is createGitArchiveSnapshot with a minimum member size, so tests +// can force several members. Members are sized from the archive's blob total, so an +// archive larger than segmentMaxMembers times the minimum still splits evenly. +func createGitArchiveSegments(ctx context.Context, git gitRepo, commitSHA, outputTarball, directoryName string, includePaths []string, subtreePrefix string, minTargetBytes int64) (segmentIndex, error) { treeish := commitSHA archiveGit := git if subtreePrefix != "" { repoRoot, err := git.repositoryRoot(ctx) if err != nil { - return err + return segmentIndex{}, err } archiveGit = newGitRepo(repoRoot) treeish = commitSHA + ":" + subtreePrefix } + total, err := archiveGit.archiveTarSize(ctx, treeish, includePaths) + if err != nil { + return segmentIndex{}, err + } + targetBytes := max(minTargetBytes, (total+segmentMaxMembers-1)/segmentMaxMembers) + args := []string{ "archive", "--format=tar", @@ -44,26 +58,67 @@ func createGitArchiveSnapshot(ctx context.Context, git gitRepo, commitSHA, outpu treeish, } args = append(args, includePaths...) + return streamGitArchive(ctx, archiveGit, args, outputTarball, directoryName, targetBytes, segmentMaxMembers) +} - out, err := os.Create(outputTarball) - if err != nil { - return fmt.Errorf("failed to create tarball: %w", err) +// archiveTarSize estimates the uncompressed tar bytes `git archive` writes for the blobs of +// treeish under includePaths, from `git ls-tree` sizes: a header block plus padded data per +// blob. Submodule entries have no blob size and take no space in the archive. +func (g gitRepo) archiveTarSize(ctx context.Context, treeish string, includePaths []string) (int64, error) { + args := []string{"ls-tree", "-r", "-l", "-z", treeish} + if len(includePaths) > 0 { + args = append(args, "--") + args = append(args, includePaths...) } - gz, err := pgzip.NewWriterLevel(out, pgzip.DefaultCompression) + out, err := g.runBytes(ctx, args...) if err != nil { - out.Close() - return err + return 0, fmt.Errorf("failed to create git archive: %w", err) } - if err := archiveGit.runTo(ctx, ctxWriter{ctx, gz}, args...); err != nil { - gz.Close() - out.Close() - return fmt.Errorf("failed to create git archive: %w", err) + + var total int64 + for record := range bytes.SplitSeq(out, []byte{0}) { + // Each record is " \t". + meta, _, found := bytes.Cut(record, []byte{'\t'}) + if !found { + continue + } + fields := strings.Fields(string(meta)) + if len(fields) != 4 || fields[3] == "-" { + continue + } + size, err := strconv.ParseInt(fields[3], 10, 64) + if err != nil { + return 0, fmt.Errorf("failed to create git archive: %w", err) + } + total += tarBlockSize + (size+tarBlockSize-1)/tarBlockSize*tarBlockSize + } + return total, nil +} + +// streamGitArchive runs git with args and writes its tar output to outputTarball as segmented +// members. +func streamGitArchive(ctx context.Context, git gitRepo, args []string, outputTarball, component string, targetBytes int64, maxMembers int) (segmentIndex, error) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + pr, pw := io.Pipe() + gitDone := make(chan error, 1) + go func() { + err := git.runTo(ctx, pw, args...) + pw.CloseWithError(err) + gitDone <- err + }() + index, err := writeTarSegments(ctx, component, outputTarball, pr, targetBytes, maxMembers) + // git pads the archive past the end-of-archive marker, so drain it before closing the reader. + if err == nil { + _, err = io.Copy(io.Discard, pr) } - if err := gz.Close(); err != nil { - out.Close() - return fmt.Errorf("failed to finalize gzip: %w", err) + // Closing the read side makes git's next write fail, so git exits if we stopped early. + pr.CloseWithError(err) + // git's own failure is the root cause when the tar side also failed. + if gitErr := <-gitDone; gitErr != nil { + return segmentIndex{}, fmt.Errorf("failed to create git archive: %w", gitErr) } - return out.Close() + return index, err } // createPlainTarball writes the given working-tree files to outputTarball as a diff --git a/cmd/air/snapshot_package_test.go b/cmd/air/snapshot_package_test.go index f4d4fda7712..cef3d0f97ac 100644 --- a/cmd/air/snapshot_package_test.go +++ b/cmd/air/snapshot_package_test.go @@ -47,7 +47,7 @@ func TestCreateGitArchiveSnapshot(t *testing.T) { out := filepath.Join(t.TempDir(), "snap.tar.gz") dirName := filepath.Base(repo) - require.NoError(t, createGitArchiveSnapshot(ctx, newGitRepo(repo), sha, out, dirName, nil, "")) + mustCreateGitArchiveSnapshot(t, ctx, newGitRepo(repo), sha, out, dirName, nil, "") entries := tarballEntries(t, out) // Every real entry is prefixed with the directory name. git archive also emits a @@ -71,7 +71,7 @@ func TestCreateGitArchiveSnapshot_IncludePaths(t *testing.T) { out := filepath.Join(t.TempDir(), "snap.tar.gz") dirName := filepath.Base(repo) - require.NoError(t, createGitArchiveSnapshot(ctx, newGitRepo(repo), sha, out, dirName, []string{"src"}, "")) + mustCreateGitArchiveSnapshot(t, ctx, newGitRepo(repo), sha, out, dirName, []string{"src"}, "") entries := tarballEntries(t, out) assert.Contains(t, entries, dirName+"/src/model.py") @@ -91,7 +91,7 @@ func TestCreateGitArchiveSnapshot_SubdirectoryRootPath(t *testing.T) { require.NoError(t, err) out := filepath.Join(t.TempDir(), "snap.tar.gz") - require.NoError(t, createGitArchiveSnapshot(ctx, newGitRepo(rootPath), sha, out, "subpkg", nil, prefix)) + mustCreateGitArchiveSnapshot(t, ctx, newGitRepo(rootPath), sha, out, "subpkg", nil, prefix) entries := tarballEntries(t, out) assert.Contains(t, entries, "subpkg/train.py") @@ -115,7 +115,7 @@ func TestCreateGitArchiveSnapshot_SubdirectoryRootPathWithIncludePaths(t *testin require.NoError(t, err) out := filepath.Join(t.TempDir(), "snap.tar.gz") - require.NoError(t, createGitArchiveSnapshot(ctx, newGitRepo(rootPath), sha, out, "subpkg", []string{"src"}, prefix)) + mustCreateGitArchiveSnapshot(t, ctx, newGitRepo(rootPath), sha, out, "subpkg", []string{"src"}, prefix) entries := tarballEntries(t, out) assert.Contains(t, entries, "subpkg/src/model.py") @@ -130,7 +130,7 @@ func TestCreateGitArchiveSnapshot_GitFailureSurfacesStderr(t *testing.T) { commitAll(t, repo, "init") out := filepath.Join(t.TempDir(), "snap.tar.gz") - err := createGitArchiveSnapshot(ctx, newGitRepo(repo), "0000000000000000000000000000000000000000", out, "repo", nil, "") + _, err := createGitArchiveSnapshot(ctx, newGitRepo(repo), "0000000000000000000000000000000000000000", out, "repo", nil, "") require.ErrorContains(t, err, "failed to create git archive") assert.ErrorContains(t, err, "fatal") } @@ -142,7 +142,7 @@ func TestCreateGitArchiveSnapshotStopsWhenCancelled(t *testing.T) { ctx, cancel := context.WithCancel(t.Context()) cancel() - err := createGitArchiveSnapshot(ctx, newGitRepo(repo), sha, filepath.Join(t.TempDir(), "snap.tar.gz"), "repo", nil, "") + _, err := createGitArchiveSnapshot(ctx, newGitRepo(repo), sha, filepath.Join(t.TempDir(), "snap.tar.gz"), "repo", nil, "") require.ErrorIs(t, err, context.Canceled) } diff --git a/cmd/air/snapshot_segment.go b/cmd/air/snapshot_segment.go index 6710fb6a6fa..4f1b95414c4 100644 --- a/cmd/air/snapshot_segment.go +++ b/cmd/air/snapshot_segment.go @@ -1,8 +1,10 @@ package aircmd import ( + "archive/tar" "compress/gzip" "context" + "errors" "fmt" "io" "os" @@ -185,6 +187,147 @@ func writeMemberTar(ctx context.Context, w io.Writer, entries []tarpack.Entry, l return nil } +// writeTarSegments streams the tar archive in tarStream to outputTarball as gzip members. The +// archive size is not known up front, so the caller passes targetBytes, the size at which a +// member closes at the next entry boundary. Once maxMembers members are open, the last one +// takes the rest. Each member is first written as raw tar to a temp file, so the stream is +// never blocked on compression. Members then compress concurrently, at most runtime.NumCPU() +// at a time, and the output is assembled only after all of them succeed. +func writeTarSegments(ctx context.Context, component, outputTarball string, tarStream io.Reader, targetBytes int64, maxMembers int) (segmentIndex, error) { + dir, err := os.MkdirTemp("", "air-snapshot-members-") + if err != nil { + return segmentIndex{}, fmt.Errorf("failed to create snapshot members dir: %w", err) + } + defer os.RemoveAll(dir) + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + g, gctx := errgroup.WithContext(ctx) + spool := &segmentSpool{dir: dir, g: g, slots: make(chan struct{}, runtime.NumCPU())} + streamErr := spool.stream(gctx, tarStream, targetBytes, maxMembers) + if streamErr != nil { + // The stream error is the root cause. Waiting only lets the compressors stop. + cancel() + _ = g.Wait() + return segmentIndex{}, streamErr + } + if err := g.Wait(); err != nil { + return segmentIndex{}, err + } + return concatMembers(component, outputTarball, spool.members) +} + +// segmentSpool writes members as raw tar files and compresses each one in the background. +type segmentSpool struct { + dir string + g *errgroup.Group + slots chan struct{} + members []string // compressed member paths, in order +} + +// rawMember is a member being written as raw tar, counting its bytes. +type rawMember struct { + raw *os.File + rawPath string + memberPath string + tar *tar.Writer + bytes int64 +} + +func (m *rawMember) Write(p []byte) (int, error) { + n, err := m.raw.Write(p) + m.bytes += int64(n) + return n, err +} + +// stream copies the entries of tarStream into members, closing a member at an entry boundary +// once it holds targetBytes, and never opening more than maxMembers. +func (s *segmentSpool) stream(ctx context.Context, tarStream io.Reader, targetBytes int64, maxMembers int) error { + cur, err := s.open() + if err != nil { + return err + } + tr := tar.NewReader(tarStream) + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return fmt.Errorf("failed to read git archive: %w", err) + } + if cur.bytes >= targetBytes && len(s.members) < maxMembers { + if err := s.close(ctx, cur, false); err != nil { + return err + } + if cur, err = s.open(); err != nil { + return err + } + } + if err := cur.tar.WriteHeader(hdr); err != nil { + return fmt.Errorf("failed to write snapshot entry %s: %w", hdr.Name, err) + } + if _, err := io.Copy(cur.tar, tr); err != nil { + return fmt.Errorf("failed to write snapshot entry %s: %w", hdr.Name, err) + } + } + return s.close(ctx, cur, true) +} + +// open starts the next member as a raw tar file. +func (s *segmentSpool) open() (*rawMember, error) { + i := len(s.members) + raw, err := os.Create(filepath.Join(s.dir, fmt.Sprintf("raw-%d", i))) + if err != nil { + return nil, fmt.Errorf("failed to create snapshot member: %w", err) + } + memberPath := filepath.Join(s.dir, fmt.Sprintf("member-%d", i)) + s.members = append(s.members, memberPath) + m := &rawMember{raw: raw, rawPath: raw.Name(), memberPath: memberPath} + m.tar = tar.NewWriter(m) + return m, nil +} + +// close finishes a raw member and queues its compression. Only the last member gets the +// end-of-archive marker. +func (s *segmentSpool) close(ctx context.Context, m *rawMember, last bool) error { + if last { + if err := m.tar.Close(); err != nil { + return fmt.Errorf("failed to finish snapshot tar: %w", err) + } + } else if err := m.tar.Flush(); err != nil { + return fmt.Errorf("failed to finish snapshot member: %w", err) + } + if err := m.raw.Close(); err != nil { + return fmt.Errorf("failed to finish snapshot member: %w", err) + } + s.g.Go(func() error { + s.slots <- struct{}{} + defer func() { <-s.slots }() + return compressRaw(ctx, m.rawPath, m.memberPath) + }) + return nil +} + +// compressRaw gzips the raw tar at rawPath into the member file at memberPath. +func compressRaw(ctx context.Context, rawPath, memberPath string) error { + raw, err := os.Open(rawPath) + if err != nil { + return err + } + defer raw.Close() + return spoolFile(ctx, memberPath, func(ctx context.Context, w io.Writer) error { + gz, err := gzip.NewWriterLevel(ctxWriter{ctx, w}, gzip.DefaultCompression) + if err != nil { + return err + } + if _, err := io.Copy(gz, raw); err != nil { + return fmt.Errorf("failed to create plain tarball: %w", err) + } + return gz.Close() + }) +} + // appendFile copies the file at path onto the end of w and returns the bytes copied. func appendFile(w io.Writer, path string) (int64, error) { f, err := os.Open(path) diff --git a/cmd/air/snapshot_upload.go b/cmd/air/snapshot_upload.go index b22362457ab..fd6903e2c75 100644 --- a/cmd/air/snapshot_upload.go +++ b/cmd/air/snapshot_upload.go @@ -139,19 +139,14 @@ func snapshotTarName(ctx context.Context, repoPath string, plan snapshotPlan) (s return fmt.Sprintf("%s_%s.tar.gz", dirName, computePlainTarKey(files)[:16]), files, nil } -// packageSnapshot writes the snapshot to tarball per the resolved plan: `git archive` -// of the pinned commit for git_archive, else a segmented plain tar of the pre-listed -// working-tree files. It returns the segment index for plain_tar and nil for -// git_archive, which has no sidecar yet. -func packageSnapshot(ctx context.Context, repoPath string, plan snapshotPlan, files []snapshotFile, tarball string) (*segmentIndex, error) { +// packageSnapshot writes the snapshot to tarball per the resolved plan, as a segmented +// tarball: the pinned commit's `git archive` for git_archive, else the pre-listed +// working-tree files. It returns the index that is uploaded alongside the tarball. +func packageSnapshot(ctx context.Context, repoPath string, plan snapshotPlan, files []snapshotFile, tarball string) (segmentIndex, error) { if plan.mode == modeGitArchive { - return nil, createGitArchiveSnapshot(ctx, newGitRepo(repoPath), plan.commitSHA, tarball, filepath.Base(repoPath), plan.includePaths, plan.subtreePrefix) + return createGitArchiveSnapshot(ctx, newGitRepo(repoPath), plan.commitSHA, tarball, filepath.Base(repoPath), plan.includePaths, plan.subtreePrefix) } - index, err := createPlainTarball(ctx, repoPath, tarball, files) - if err != nil { - return nil, err - } - return &index, nil + return createPlainTarball(ctx, repoPath, tarball, files) } // airArtifactInternalDir mirrors bundle/libraries.InternalDirName: CLI-uploaded @@ -180,7 +175,7 @@ func uploadSnapshotTarball(ctx context.Context, w *databricks.WorkspaceClient, r // upload the bytes now or reuse an object already in the store. remote := path.Join(uploadPath, tarName) - cached, err := snapshotCached(ctx, f, tarName, plan.mode == modePlainTar) + cached, err := snapshotCached(ctx, f, tarName) if err != nil { return result, err } @@ -223,9 +218,7 @@ func uploadSnapshotTarball(ctx context.Context, w *databricks.WorkspaceClient, r // The sidecar and tarball are written concurrently. The job is only submitted once both // succeed, and a later cache hit requires both (see snapshotCached). g, gctx := errgroup.WithContext(ctx) - if index != nil { - g.Go(func() error { return uploadSegmentIndex(gctx, f, tarName, index) }) - } + g.Go(func() error { return uploadSegmentIndex(gctx, f, tarName, &index) }) g.Go(func() error { if err := f.Write(gctx, tarName, file, filer.OverwriteIfExists, filer.CreateParentDirectories); err != nil { return fmt.Errorf("failed to upload snapshot %s: %w", tarName, err) @@ -241,14 +234,11 @@ func uploadSnapshotTarball(ctx context.Context, w *databricks.WorkspaceClient, r return result, nil } -// snapshotCached reports whether the snapshot is already in the store. For an indexed -// (plain_tar) snapshot both the tarball and its index sidecar must exist: a half-written -// pair from an interrupted upload is a miss, and the pair is re-uploaded. -func snapshotCached(ctx context.Context, store filer.Filer, tarName string, indexed bool) (bool, error) { - names := []string{tarName} - if indexed { - names = append(names, tarName+segmentIndexSuffix) - } +// snapshotCached reports whether the snapshot is already in the store. Both the tarball and +// its index sidecar must exist: a half-written pair from an interrupted upload is a miss, and +// the pair is re-uploaded. A tarball from before sidecars were written is therefore also a miss. +func snapshotCached(ctx context.Context, store filer.Filer, tarName string) (bool, error) { + names := []string{tarName, tarName + segmentIndexSuffix} exists := make([]bool, len(names)) g, gctx := errgroup.WithContext(ctx) for i, name := range names { diff --git a/cmd/air/testdata/cache_keys.json b/cmd/air/testdata/cache_keys.json index 06673499109..23ceba62c02 100644 --- a/cmd/air/testdata/cache_keys.json +++ b/cmd/air/testdata/cache_keys.json @@ -3,13 +3,13 @@ "name": "commit_no_paths", "commit_sha": "a3492b801c0ffee00000000000000000000dead", "include_paths": null, - "cache_key": "8d7bc445ac83dfe432a353718ae8b1ae40eb00c860352662e6df96b7fdf862a5" + "cache_key": "b372b0c84900bb4dc5a3dbe42f0ed0fc3a30a39096d0f1671ec7a732963880b3" }, { "name": "commit_empty_paths_none", "commit_sha": "0000000000000000000000000000000000000000", "include_paths": null, - "cache_key": "a07e0fa4d38f3c2d08c3ff6bbff0158cea01796aaaf5368b0d0e247cebd27c80" + "cache_key": "8c893a124d4c92ba09d6d69e459e3f1ee3704d18c8d3de49a6b82b2e8ac1332e" }, { "name": "single_path", @@ -17,7 +17,7 @@ "include_paths": [ "research" ], - "cache_key": "97bd562ad591af23b9a05651d88c01abbbcc24632e9a8f2534230a61492b9e0a" + "cache_key": "e1cb84d3e4cf51861a13e9f5693a5b842809f622120bf665529e52f6da58f96f" }, { "name": "multi_path_sorted", @@ -27,7 +27,7 @@ "b", "c" ], - "cache_key": "17d021577e3615e9a603a2f701f918d74ad16218efc4b2950ee20f6a18b1a174" + "cache_key": "6421973342e0e6a5fe15704d664372deb6b12c6d111e1d165c6916b49f0b44fc" }, { "name": "multi_path_unsorted", @@ -37,7 +37,7 @@ "a", "b" ], - "cache_key": "17d021577e3615e9a603a2f701f918d74ad16218efc4b2950ee20f6a18b1a174" + "cache_key": "6421973342e0e6a5fe15704d664372deb6b12c6d111e1d165c6916b49f0b44fc" }, { "name": "path_whitespace", @@ -46,7 +46,7 @@ " research ", " data " ], - "cache_key": "c6a6fa1d866ac90a5f371610ee08efa8aeb061a43b8a4acae30765db21f4155b" + "cache_key": "c6354f4b4f2c02e559926a079b6eb3fc560803cca426deb016398c56ee41f8ec" }, { "name": "nested_paths", @@ -56,13 +56,13 @@ "src/data", "README.md" ], - "cache_key": "2b18452b1019186a8e5b34af540a0545f96f24bc61771f58fa71fc218cb0275c" + "cache_key": "489892a2fdd3b869069b77f9e3b6dee5d51d85b649135b6654c3858305526182" }, { "name": "short_sha", "commit_sha": "a3492b8", "include_paths": null, - "cache_key": "16be4d7d83ecd01b8d3d274dafb07b3bccc72cf7f7299578e634cd01e1825345" + "cache_key": "5d0dc33af0fc697db6657113ecaf1b51cf8837a989c62604ba6c22852c8a900d" }, { "name": "dup_paths", @@ -72,6 +72,6 @@ "x", "y" ], - "cache_key": "407a5d8c3a0e9438d5abf653c4d3a43928c300b5630e2f83f435661b9fbd599e" + "cache_key": "20af9cc2df7443ce5e07e901337c9677185612fbedbd5e0e68272bbcef9c97bb" } ]