diff --git a/.nextchanges/air/git-archive-pgzip.md b/.nextchanges/air/git-archive-pgzip.md
new file mode 100644
index 00000000000..f4a9820e728
--- /dev/null
+++ b/.nextchanges/air/git-archive-pgzip.md
@@ -0,0 +1 @@
+* Speed up `databricks air run` for git-pinned `code_source` snapshots by compressing the archive in parallel, which substantially shortens packaging on large repositories. ([#7402](https://github.com/databricks/cli/pull/7402))
diff --git a/.nextchanges/air/git-segmented-snapshot.md b/.nextchanges/air/git-segmented-snapshot.md
new file mode 100644
index 00000000000..af2a7b255a8
--- /dev/null
+++ b/.nextchanges/air/git-segmented-snapshot.md
@@ -0,0 +1 @@
+* `databricks air run` with a pinned git commit packages the snapshot as parallel gzip segments with an index, which shortens submission for large repositories. ([#7404](https://github.com/databricks/cli/pull/7404))
diff --git a/cmd/air/runsubmit_test.go b/cmd/air/runsubmit_test.go
index 98bd7f9a4e0..e6eb46e4491 100644
--- a/cmd/air/runsubmit_test.go
+++ b/cmd/air/runsubmit_test.go
@@ -904,11 +904,14 @@ func TestSubmitWorkloadGitArchiveCaching(t *testing.T) {
// persistence so the second submit's cache-existence Stat sees the first upload.
// Dedupe by path: the filer mkdirs-and-retries the import on a missing
// parent dir, so one logical upload can hit this route more than once.
+ var mu sync.Mutex
uploaded := map[string]bool{}
server.Handle("POST", "/api/2.0/workspace-files/import-file/{path...}", func(req testserver.Request) any {
p := req.Vars["path"]
if strings.Contains(p, "/.air/repo_snapshots/") {
+ mu.Lock()
uploaded[p] = true
+ mu.Unlock()
}
return req.Workspace.WorkspaceFilesImportFile(p, req.Body, req.URL.Query().Get("overwrite") == "true")
})
@@ -943,7 +946,7 @@ code_source:
// Same pinned commit → identical content-addressed remote path, uploaded once
// (the second submit is a cache hit and moves no bytes).
assert.Equal(t, first.CodeSourcePath, second.CodeSourcePath)
- assert.Len(t, uploaded, 1, "git_archive cache hit should skip the second upload")
+ assert.Len(t, uploaded, 2, "git_archive cache hit should skip the second upload: tarball and index sidecar")
require.NotNil(t, first.SizeBytes)
assert.Positive(t, *first.SizeBytes)
assert.Nil(t, second.SizeBytes)
diff --git a/cmd/air/snapshot_cachekey.go b/cmd/air/snapshot_cachekey.go
index 7b278a3a8cf..b130b17b56d 100644
--- a/cmd/air/snapshot_cachekey.go
+++ b/cmd/air/snapshot_cachekey.go
@@ -14,7 +14,7 @@ import (
)
// snapshotPackagingVersion is bumped when packaging logic changes in a way that invalidates existing caches
-const snapshotPackagingVersion = "v1"
+const snapshotPackagingVersion = "v2"
// plainTarKeyVersion namespaces the plain_tar working-tree key (so it can never collide
// with a git_archive key) and lets us invalidate it if the fingerprint scheme changes.
diff --git a/cmd/air/snapshot_git.go b/cmd/air/snapshot_git.go
index 6172ccec9c6..69972dd4d2b 100644
--- a/cmd/air/snapshot_git.go
+++ b/cmd/air/snapshot_git.go
@@ -6,6 +6,7 @@ import (
"encoding/json"
"errors"
"fmt"
+ "io"
"os/exec"
"strings"
"time"
@@ -35,20 +36,30 @@ func (g gitRepo) run(ctx context.Context, args ...string) (string, error) {
// runBytes is run returning raw stdout bytes, for the dirty-diff capture which needs
// exact bytes and a size measurement.
func (g gitRepo) runBytes(ctx context.Context, args ...string) ([]byte, error) {
+ var stdout bytes.Buffer
+ if err := g.runTo(ctx, &stdout, args...); err != nil {
+ return nil, err
+ }
+ return stdout.Bytes(), nil
+}
+
+// runTo is run streaming stdout into w as git produces it, for output too large to
+// buffer such as `git archive`.
+func (g gitRepo) runTo(ctx context.Context, w io.Writer, args ...string) error {
full := append([]string{"-C", g.path}, args...)
cmd := exec.CommandContext(ctx, "git", full...)
- var stdout, stderr bytes.Buffer
- cmd.Stdout = &stdout
+ var stderr bytes.Buffer
+ cmd.Stdout = w
cmd.Stderr = &stderr
if err := cmd.Run(); err != nil {
msg := strings.TrimSpace(stderr.String())
if msg == "" {
- return nil, fmt.Errorf("git %s: %w", strings.Join(args, " "), err)
+ return fmt.Errorf("git %s: %w", strings.Join(args, " "), err)
}
- return nil, fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, msg)
+ return fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, msg)
}
- return stdout.Bytes(), nil
+ return nil
}
// isRepository reports whether the path is inside a git work tree. Using
diff --git a/cmd/air/snapshot_git_segment_test.go b/cmd/air/snapshot_git_segment_test.go
new file mode 100644
index 00000000000..e6955eeab8f
--- /dev/null
+++ b/cmd/air/snapshot_git_segment_test.go
@@ -0,0 +1,105 @@
+package aircmd
+
+import (
+ "bytes"
+ "compress/gzip"
+ "context"
+ "fmt"
+ "io"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// mustCreateGitArchiveSnapshot packages commitSHA with createGitArchiveSnapshot and fails the
+// test on error.
+func mustCreateGitArchiveSnapshot(t *testing.T, ctx context.Context, git gitRepo, commitSHA, outputTarball, directoryName string, includePaths []string, subtreePrefix string) {
+ t.Helper()
+ _, err := createGitArchiveSnapshot(ctx, git, commitSHA, outputTarball, directoryName, includePaths, subtreePrefix)
+ require.NoError(t, err)
+}
+
+// commitFiles commits n files of size bytes each, spread over three directories.
+func commitFiles(t *testing.T, n, size int) (string, string) {
+ t.Helper()
+ repo := newTestRepo(t)
+ for i := range n {
+ writeRepoFile(t, repo, fmt.Sprintf("pkg%d/file%d.py", i%3, i), strings.Repeat("x", size+i))
+ }
+ return repo, commitAll(t, repo, "init")
+}
+
+// A multi-member git_archive snapshot reads as one stream with pax_global_header first. Its
+// members are contiguous, decode independently to the same entries, and cover the file.
+func TestCreateGitArchiveSnapshotSegmented(t *testing.T) {
+ repo, sha := commitFiles(t, 30, 200)
+ dirName := filepath.Base(repo)
+
+ out := filepath.Join(t.TempDir(), "snap.tar.gz")
+ index, err := createGitArchiveSegments(t.Context(), newGitRepo(repo), sha, out, dirName, nil, "", 4096)
+ require.NoError(t, err)
+ require.Greater(t, len(index.Members), 1)
+ assert.Equal(t, dirName, index.Component)
+
+ data, err := os.ReadFile(out)
+ require.NoError(t, err)
+ assert.Equal(t, int64(len(data)), index.Size)
+
+ gz, err := gzip.NewReader(bytes.NewReader(data))
+ require.NoError(t, err)
+ single := tarNames(t, gz)
+ require.NotEmpty(t, single)
+ assert.Equal(t, "pax_global_header", single[0])
+
+ var union []string
+ var next int64
+ for i, m := range index.Members {
+ assert.Equal(t, next, m.Offset, "member %d not contiguous", i)
+ next += m.Length
+ member, err := gzip.NewReader(bytes.NewReader(data[m.Offset : m.Offset+m.Length]))
+ require.NoError(t, err)
+ names := tarNames(t, member)
+ if i == 0 {
+ require.NotEmpty(t, names)
+ assert.Equal(t, "pax_global_header", names[0])
+ }
+ union = append(union, names...)
+ }
+ assert.Equal(t, index.Size, next)
+ assert.Equal(t, single, union)
+}
+
+// An archive larger than maxMembers times the minimum target still splits into balanced
+// members, instead of the last member absorbing the rest. Balance is checked on uncompressed
+// tar bytes, which is what the launcher parallelizes over.
+func TestCreateGitArchiveSnapshotBalancesLargeArchive(t *testing.T) {
+ const minTarget = 4096
+ // 64 files of about 4 KiB: about 256 KiB of blobs, well over 16 times the 4 KiB minimum.
+ repo, sha := commitFiles(t, 64, 4000)
+ out := filepath.Join(t.TempDir(), "snap.tar.gz")
+ index, err := createGitArchiveSegments(t.Context(), newGitRepo(repo), sha, out, filepath.Base(repo), nil, "", minTarget)
+ require.NoError(t, err)
+ require.LessOrEqual(t, len(index.Members), segmentMaxMembers)
+ require.Greater(t, len(index.Members), 1)
+
+ data, err := os.ReadFile(out)
+ require.NoError(t, err)
+ var sizes []int64
+ var total int64
+ for _, m := range index.Members {
+ gz, err := gzip.NewReader(bytes.NewReader(data[m.Offset : m.Offset+m.Length]))
+ require.NoError(t, err)
+ n, err := io.Copy(io.Discard, gz)
+ require.NoError(t, err)
+ sizes = append(sizes, n)
+ total += n
+ }
+ average := total / int64(len(sizes))
+ for i, n := range sizes {
+ assert.LessOrEqual(t, n, 2*average, "member %d holds %d of %d uncompressed bytes", i, n, total)
+ }
+}
diff --git a/cmd/air/snapshot_package.go b/cmd/air/snapshot_package.go
index 609c07950ad..c86ddbcc58d 100644
--- a/cmd/air/snapshot_package.go
+++ b/cmd/air/snapshot_package.go
@@ -9,6 +9,7 @@ import (
"os"
"os/exec"
"path/filepath"
+ "strconv"
"strings"
"github.com/databricks/cli/libs/tarpack"
@@ -20,33 +21,104 @@ import (
// load-bearing — the remote entry_script extracts to /databricks/code_source/
— so
// both forms prefix every entry with it.
-// createGitArchiveSnapshot writes a gzipped tar of commitSHA to outputTarball via
-// `git archive`, with every entry prefixed by directoryName/. When subtreePrefix is
-// set, only that repository subtree is archived and includePaths are relative to it.
-func createGitArchiveSnapshot(ctx context.Context, git gitRepo, commitSHA, outputTarball, directoryName string, includePaths []string, subtreePrefix string) error {
+// createGitArchiveSnapshot writes commitSHA to outputTarball as a segmented tarball (see
+// snapshot_segment.go) and returns its index. Every entry is prefixed by directoryName/.
+// The tar is streamed from `git archive --format=tar`, since git's own tar.gz format
+// compresses on one thread. When subtreePrefix is set, only that repository subtree is
+// archived and includePaths are relative to it.
+func createGitArchiveSnapshot(ctx context.Context, git gitRepo, commitSHA, outputTarball, directoryName string, includePaths []string, subtreePrefix string) (segmentIndex, error) {
+ return createGitArchiveSegments(ctx, git, commitSHA, outputTarball, directoryName, includePaths, subtreePrefix, segmentTargetBytes)
+}
+
+// createGitArchiveSegments is createGitArchiveSnapshot with a minimum member size, so tests
+// can force several members. Members are sized from the archive's blob total, so an
+// archive larger than segmentMaxMembers times the minimum still splits evenly.
+func createGitArchiveSegments(ctx context.Context, git gitRepo, commitSHA, outputTarball, directoryName string, includePaths []string, subtreePrefix string, minTargetBytes int64) (segmentIndex, error) {
treeish := commitSHA
archiveGit := git
if subtreePrefix != "" {
repoRoot, err := git.repositoryRoot(ctx)
if err != nil {
- return err
+ return segmentIndex{}, err
}
archiveGit = newGitRepo(repoRoot)
treeish = commitSHA + ":" + subtreePrefix
}
+ total, err := archiveGit.archiveTarSize(ctx, treeish, includePaths)
+ if err != nil {
+ return segmentIndex{}, err
+ }
+ targetBytes := max(minTargetBytes, (total+segmentMaxMembers-1)/segmentMaxMembers)
+
args := []string{
"archive",
- "--format=tar.gz",
+ "--format=tar",
"--prefix=" + directoryName + "/",
- "-o", outputTarball,
treeish,
}
args = append(args, includePaths...)
- if _, err := archiveGit.run(ctx, args...); err != nil {
- return fmt.Errorf("failed to create git archive: %w", err)
+ return streamGitArchive(ctx, archiveGit, args, outputTarball, directoryName, targetBytes, segmentMaxMembers)
+}
+
+// archiveTarSize estimates the uncompressed tar bytes `git archive` writes for the blobs of
+// treeish under includePaths, from `git ls-tree` sizes: a header block plus padded data per
+// blob. Submodule entries have no blob size and take no space in the archive.
+func (g gitRepo) archiveTarSize(ctx context.Context, treeish string, includePaths []string) (int64, error) {
+ args := []string{"ls-tree", "-r", "-l", "-z", treeish}
+ if len(includePaths) > 0 {
+ args = append(args, "--")
+ args = append(args, includePaths...)
+ }
+ out, err := g.runBytes(ctx, args...)
+ if err != nil {
+ return 0, fmt.Errorf("failed to create git archive: %w", err)
+ }
+
+ var total int64
+ for record := range bytes.SplitSeq(out, []byte{0}) {
+ // Each record is "