Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/air/git-archive-pgzip.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* Speed up `databricks air run` for git-pinned `code_source` snapshots by compressing the archive in parallel, which substantially shortens packaging on large repositories. ([#7402](https://github.com/databricks/cli/pull/7402))
1 change: 1 addition & 0 deletions .nextchanges/air/git-segmented-snapshot.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* `databricks air run` with a pinned git commit packages the snapshot as parallel gzip segments with an index, which shortens submission for large repositories. ([#7404](https://github.com/databricks/cli/pull/7404))
5 changes: 4 additions & 1 deletion cmd/air/runsubmit_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -904,11 +904,14 @@ func TestSubmitWorkloadGitArchiveCaching(t *testing.T) {
// persistence so the second submit's cache-existence Stat sees the first upload.
// Dedupe by path: the filer mkdirs-and-retries the import on a missing
// parent dir, so one logical upload can hit this route more than once.
var mu sync.Mutex
uploaded := map[string]bool{}
server.Handle("POST", "/api/2.0/workspace-files/import-file/{path...}", func(req testserver.Request) any {
p := req.Vars["path"]
if strings.Contains(p, "/.air/repo_snapshots/") {
mu.Lock()
uploaded[p] = true
mu.Unlock()
}
return req.Workspace.WorkspaceFilesImportFile(p, req.Body, req.URL.Query().Get("overwrite") == "true")
})
Expand Down Expand Up @@ -943,7 +946,7 @@ code_source:
// Same pinned commit → identical content-addressed remote path, uploaded once
// (the second submit is a cache hit and moves no bytes).
assert.Equal(t, first.CodeSourcePath, second.CodeSourcePath)
assert.Len(t, uploaded, 1, "git_archive cache hit should skip the second upload")
assert.Len(t, uploaded, 2, "git_archive cache hit should skip the second upload: tarball and index sidecar")
require.NotNil(t, first.SizeBytes)
assert.Positive(t, *first.SizeBytes)
assert.Nil(t, second.SizeBytes)
Expand Down
2 changes: 1 addition & 1 deletion cmd/air/snapshot_cachekey.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import (
)

// snapshotPackagingVersion is bumped when packaging logic changes in a way that invalidates existing caches
const snapshotPackagingVersion = "v1"
const snapshotPackagingVersion = "v2"

// plainTarKeyVersion namespaces the plain_tar working-tree key (so it can never collide
// with a git_archive key) and lets us invalidate it if the fingerprint scheme changes.
Expand Down
21 changes: 16 additions & 5 deletions cmd/air/snapshot_git.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"encoding/json"
"errors"
"fmt"
"io"
"os/exec"
"strings"
"time"
Expand Down Expand Up @@ -35,20 +36,30 @@ func (g gitRepo) run(ctx context.Context, args ...string) (string, error) {
// runBytes is run returning raw stdout bytes, for the dirty-diff capture which needs
// exact bytes and a size measurement.
func (g gitRepo) runBytes(ctx context.Context, args ...string) ([]byte, error) {
var stdout bytes.Buffer
if err := g.runTo(ctx, &stdout, args...); err != nil {
return nil, err
}
return stdout.Bytes(), nil
}

// runTo is run streaming stdout into w as git produces it, for output too large to
// buffer such as `git archive`.
func (g gitRepo) runTo(ctx context.Context, w io.Writer, args ...string) error {
full := append([]string{"-C", g.path}, args...)
cmd := exec.CommandContext(ctx, "git", full...)

var stdout, stderr bytes.Buffer
cmd.Stdout = &stdout
var stderr bytes.Buffer
cmd.Stdout = w
cmd.Stderr = &stderr
if err := cmd.Run(); err != nil {
msg := strings.TrimSpace(stderr.String())
if msg == "" {
return nil, fmt.Errorf("git %s: %w", strings.Join(args, " "), err)
return fmt.Errorf("git %s: %w", strings.Join(args, " "), err)
}
return nil, fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, msg)
return fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, msg)
}
return stdout.Bytes(), nil
return nil
}

// isRepository reports whether the path is inside a git work tree. Using
Expand Down
105 changes: 105 additions & 0 deletions cmd/air/snapshot_git_segment_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
package aircmd

import (
"bytes"
"compress/gzip"
"context"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

// mustCreateGitArchiveSnapshot packages commitSHA with createGitArchiveSnapshot and fails the
// test on error.
func mustCreateGitArchiveSnapshot(t *testing.T, ctx context.Context, git gitRepo, commitSHA, outputTarball, directoryName string, includePaths []string, subtreePrefix string) {
t.Helper()
_, err := createGitArchiveSnapshot(ctx, git, commitSHA, outputTarball, directoryName, includePaths, subtreePrefix)
require.NoError(t, err)
}

// commitFiles commits n files of size bytes each, spread over three directories.
func commitFiles(t *testing.T, n, size int) (string, string) {
t.Helper()
repo := newTestRepo(t)
for i := range n {
writeRepoFile(t, repo, fmt.Sprintf("pkg%d/file%d.py", i%3, i), strings.Repeat("x", size+i))
}
return repo, commitAll(t, repo, "init")
}

// A multi-member git_archive snapshot reads as one stream with pax_global_header first. Its
// members are contiguous, decode independently to the same entries, and cover the file.
func TestCreateGitArchiveSnapshotSegmented(t *testing.T) {
repo, sha := commitFiles(t, 30, 200)
dirName := filepath.Base(repo)

out := filepath.Join(t.TempDir(), "snap.tar.gz")
index, err := createGitArchiveSegments(t.Context(), newGitRepo(repo), sha, out, dirName, nil, "", 4096)
require.NoError(t, err)
require.Greater(t, len(index.Members), 1)
assert.Equal(t, dirName, index.Component)

data, err := os.ReadFile(out)
require.NoError(t, err)
assert.Equal(t, int64(len(data)), index.Size)

gz, err := gzip.NewReader(bytes.NewReader(data))
require.NoError(t, err)
single := tarNames(t, gz)
require.NotEmpty(t, single)
assert.Equal(t, "pax_global_header", single[0])

var union []string
var next int64
for i, m := range index.Members {
assert.Equal(t, next, m.Offset, "member %d not contiguous", i)
next += m.Length
member, err := gzip.NewReader(bytes.NewReader(data[m.Offset : m.Offset+m.Length]))
require.NoError(t, err)
names := tarNames(t, member)
if i == 0 {
require.NotEmpty(t, names)
assert.Equal(t, "pax_global_header", names[0])
}
union = append(union, names...)
}
assert.Equal(t, index.Size, next)
assert.Equal(t, single, union)
}

// An archive larger than maxMembers times the minimum target still splits into balanced
// members, instead of the last member absorbing the rest. Balance is checked on uncompressed
// tar bytes, which is what the launcher parallelizes over.
func TestCreateGitArchiveSnapshotBalancesLargeArchive(t *testing.T) {
const minTarget = 4096
// 64 files of about 4 KiB: about 256 KiB of blobs, well over 16 times the 4 KiB minimum.
repo, sha := commitFiles(t, 64, 4000)
out := filepath.Join(t.TempDir(), "snap.tar.gz")
index, err := createGitArchiveSegments(t.Context(), newGitRepo(repo), sha, out, filepath.Base(repo), nil, "", minTarget)
require.NoError(t, err)
require.LessOrEqual(t, len(index.Members), segmentMaxMembers)
require.Greater(t, len(index.Members), 1)

data, err := os.ReadFile(out)
require.NoError(t, err)
var sizes []int64
var total int64
for _, m := range index.Members {
gz, err := gzip.NewReader(bytes.NewReader(data[m.Offset : m.Offset+m.Length]))
require.NoError(t, err)
n, err := io.Copy(io.Discard, gz)
require.NoError(t, err)
sizes = append(sizes, n)
total += n
}
average := total / int64(len(sizes))
for i, n := range sizes {
assert.LessOrEqual(t, n, 2*average, "member %d holds %d of %d uncompressed bytes", i, n, total)
}
}
92 changes: 82 additions & 10 deletions cmd/air/snapshot_package.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"

"github.com/databricks/cli/libs/tarpack"
Expand All @@ -20,33 +21,104 @@ import (
// load-bearing — the remote entry_script extracts to /databricks/code_source/<dir> — so
// both forms prefix every entry with it.

// createGitArchiveSnapshot writes a gzipped tar of commitSHA to outputTarball via
// `git archive`, with every entry prefixed by directoryName/. When subtreePrefix is
// set, only that repository subtree is archived and includePaths are relative to it.
func createGitArchiveSnapshot(ctx context.Context, git gitRepo, commitSHA, outputTarball, directoryName string, includePaths []string, subtreePrefix string) error {
// createGitArchiveSnapshot writes commitSHA to outputTarball as a segmented tarball (see
// snapshot_segment.go) and returns its index. Every entry is prefixed by directoryName/.
// The tar is streamed from `git archive --format=tar`, since git's own tar.gz format
// compresses on one thread. When subtreePrefix is set, only that repository subtree is
// archived and includePaths are relative to it.
func createGitArchiveSnapshot(ctx context.Context, git gitRepo, commitSHA, outputTarball, directoryName string, includePaths []string, subtreePrefix string) (segmentIndex, error) {
return createGitArchiveSegments(ctx, git, commitSHA, outputTarball, directoryName, includePaths, subtreePrefix, segmentTargetBytes)
}

// createGitArchiveSegments is createGitArchiveSnapshot with a minimum member size, so tests
// can force several members. Members are sized from the archive's blob total, so an
// archive larger than segmentMaxMembers times the minimum still splits evenly.
func createGitArchiveSegments(ctx context.Context, git gitRepo, commitSHA, outputTarball, directoryName string, includePaths []string, subtreePrefix string, minTargetBytes int64) (segmentIndex, error) {
treeish := commitSHA
archiveGit := git
if subtreePrefix != "" {
repoRoot, err := git.repositoryRoot(ctx)
if err != nil {
return err
return segmentIndex{}, err
}
archiveGit = newGitRepo(repoRoot)
treeish = commitSHA + ":" + subtreePrefix
}

total, err := archiveGit.archiveTarSize(ctx, treeish, includePaths)
if err != nil {
return segmentIndex{}, err
}
targetBytes := max(minTargetBytes, (total+segmentMaxMembers-1)/segmentMaxMembers)

args := []string{
"archive",
"--format=tar.gz",
"--format=tar",
"--prefix=" + directoryName + "/",
"-o", outputTarball,
treeish,
}
args = append(args, includePaths...)
if _, err := archiveGit.run(ctx, args...); err != nil {
return fmt.Errorf("failed to create git archive: %w", err)
return streamGitArchive(ctx, archiveGit, args, outputTarball, directoryName, targetBytes, segmentMaxMembers)
}

// archiveTarSize estimates the uncompressed tar bytes `git archive` writes for the blobs of
// treeish under includePaths, from `git ls-tree` sizes: a header block plus padded data per
// blob. Submodule entries have no blob size and take no space in the archive.
func (g gitRepo) archiveTarSize(ctx context.Context, treeish string, includePaths []string) (int64, error) {
args := []string{"ls-tree", "-r", "-l", "-z", treeish}
if len(includePaths) > 0 {
args = append(args, "--")
args = append(args, includePaths...)
}
out, err := g.runBytes(ctx, args...)
if err != nil {
return 0, fmt.Errorf("failed to create git archive: %w", err)
}

var total int64
for record := range bytes.SplitSeq(out, []byte{0}) {
// Each record is "<mode> <type> <object> <size>\t<path>".
meta, _, found := bytes.Cut(record, []byte{'\t'})
if !found {
continue
}
fields := strings.Fields(string(meta))
if len(fields) != 4 || fields[3] == "-" {
continue
}
size, err := strconv.ParseInt(fields[3], 10, 64)
if err != nil {
return 0, fmt.Errorf("failed to create git archive: %w", err)
}
total += tarBlockSize + (size+tarBlockSize-1)/tarBlockSize*tarBlockSize
}
return total, nil
}

// streamGitArchive runs git with args and writes its tar output to outputTarball as segmented
// members.
func streamGitArchive(ctx context.Context, git gitRepo, args []string, outputTarball, component string, targetBytes int64, maxMembers int) (segmentIndex, error) {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
pr, pw := io.Pipe()
gitDone := make(chan error, 1)
go func() {
err := git.runTo(ctx, pw, args...)
pw.CloseWithError(err)
gitDone <- err
}()
index, err := writeTarSegments(ctx, component, outputTarball, pr, targetBytes, maxMembers)
// git pads the archive past the end-of-archive marker, so drain it before closing the reader.
if err == nil {
_, err = io.Copy(io.Discard, pr)
}
// Closing the read side makes git's next write fail, so git exits if we stopped early.
pr.CloseWithError(err)
// git's own failure is the root cause when the tar side also failed.
if gitErr := <-gitDone; gitErr != nil {
return segmentIndex{}, fmt.Errorf("failed to create git archive: %w", gitErr)
}
return nil
return index, err
}

// createPlainTarball writes the given working-tree files to outputTarball as a
Expand Down
31 changes: 27 additions & 4 deletions cmd/air/snapshot_package_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ func TestCreateGitArchiveSnapshot(t *testing.T) {

out := filepath.Join(t.TempDir(), "snap.tar.gz")
dirName := filepath.Base(repo)
require.NoError(t, createGitArchiveSnapshot(ctx, newGitRepo(repo), sha, out, dirName, nil, ""))
mustCreateGitArchiveSnapshot(t, ctx, newGitRepo(repo), sha, out, dirName, nil, "")

entries := tarballEntries(t, out)
// Every real entry is prefixed with the directory name. git archive also emits a
Expand All @@ -71,7 +71,7 @@ func TestCreateGitArchiveSnapshot_IncludePaths(t *testing.T) {

out := filepath.Join(t.TempDir(), "snap.tar.gz")
dirName := filepath.Base(repo)
require.NoError(t, createGitArchiveSnapshot(ctx, newGitRepo(repo), sha, out, dirName, []string{"src"}, ""))
mustCreateGitArchiveSnapshot(t, ctx, newGitRepo(repo), sha, out, dirName, []string{"src"}, "")

entries := tarballEntries(t, out)
assert.Contains(t, entries, dirName+"/src/model.py")
Expand All @@ -91,7 +91,7 @@ func TestCreateGitArchiveSnapshot_SubdirectoryRootPath(t *testing.T) {
require.NoError(t, err)

out := filepath.Join(t.TempDir(), "snap.tar.gz")
require.NoError(t, createGitArchiveSnapshot(ctx, newGitRepo(rootPath), sha, out, "subpkg", nil, prefix))
mustCreateGitArchiveSnapshot(t, ctx, newGitRepo(rootPath), sha, out, "subpkg", nil, prefix)

entries := tarballEntries(t, out)
assert.Contains(t, entries, "subpkg/train.py")
Expand All @@ -115,14 +115,37 @@ func TestCreateGitArchiveSnapshot_SubdirectoryRootPathWithIncludePaths(t *testin
require.NoError(t, err)

out := filepath.Join(t.TempDir(), "snap.tar.gz")
require.NoError(t, createGitArchiveSnapshot(ctx, newGitRepo(rootPath), sha, out, "subpkg", []string{"src"}, prefix))
mustCreateGitArchiveSnapshot(t, ctx, newGitRepo(rootPath), sha, out, "subpkg", []string{"src"}, prefix)

entries := tarballEntries(t, out)
assert.Contains(t, entries, "subpkg/src/model.py")
assert.NotContains(t, entries, "subpkg/train.py")
assert.NotContains(t, entries, "subpkg/configs/train.yaml")
}

func TestCreateGitArchiveSnapshot_GitFailureSurfacesStderr(t *testing.T) {
ctx := t.Context()
repo := newTestRepo(t)
writeRepoFile(t, repo, "a.txt", "1")
commitAll(t, repo, "init")

out := filepath.Join(t.TempDir(), "snap.tar.gz")
_, err := createGitArchiveSnapshot(ctx, newGitRepo(repo), "0000000000000000000000000000000000000000", out, "repo", nil, "")
require.ErrorContains(t, err, "failed to create git archive")
assert.ErrorContains(t, err, "fatal")
}

func TestCreateGitArchiveSnapshotStopsWhenCancelled(t *testing.T) {
repo := newTestRepo(t)
writeRepoFile(t, repo, "a.txt", "1")
sha := commitAll(t, repo, "init")

ctx, cancel := context.WithCancel(t.Context())
cancel()
_, err := createGitArchiveSnapshot(ctx, newGitRepo(repo), sha, filepath.Join(t.TempDir(), "snap.tar.gz"), "repo", nil, "")
require.ErrorIs(t, err, context.Canceled)
}

func TestCreatePlainTarball(t *testing.T) {
ctx := t.Context()
repo := t.TempDir()
Expand Down
Loading
Loading