Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/air/segmented-snapshot.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* `databricks air run` packages working-tree code snapshots as parallel gzip segments with an index, which shortens submission for large code directories. ([#7403](https://github.com/databricks/cli/pull/7403))
10 changes: 5 additions & 5 deletions cmd/air/runsubmit_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -832,11 +832,11 @@ func TestSubmitWorkloadPlainTarContentAddressed(t *testing.T) {
// path) is what proves the skip: both submits resolve to the same content-addressed
// name, so a set could not tell a skipped second submit from one that re-uploaded to
// that same path.
snapshotUploads := 0
var snapshotUploads atomic.Int32
server.Handle("POST", "/api/2.0/workspace-files/import-file/{path...}", func(req testserver.Request) any {
p := req.Vars["path"]
if strings.Contains(p, "/.air/repo_snapshots/") {
snapshotUploads++
snapshotUploads.Add(1)
}
return req.Workspace.WorkspaceFilesImportFile(p, req.Body, req.URL.Query().Get("overwrite") == "true")
})
Expand Down Expand Up @@ -864,8 +864,8 @@ code_source:
sidecarStore, sidecarBase := testSidecarStore(t, w)
first, err := uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase)
require.NoError(t, err)
require.NotZero(t, snapshotUploads, "first submit should upload the tarball")
afterFirst := snapshotUploads
require.NotZero(t, snapshotUploads.Load(), "first submit should upload the tarball")
afterFirst := snapshotUploads.Load()
second, err := uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase)
require.NoError(t, err)

Expand All @@ -877,7 +877,7 @@ code_source:
// Same unchanged tree → identical remote path, and the second submit moved no bytes:
// zero new import-file calls (a real skip), not a re-upload to the same name.
assert.Equal(t, first.CodeSourcePath, second.CodeSourcePath)
assert.Equal(t, afterFirst, snapshotUploads, "unchanged plain_tar should skip the second upload")
assert.Equal(t, afterFirst, snapshotUploads.Load(), "unchanged plain_tar should skip the second upload")
require.NotNil(t, first.SizeBytes)
assert.Positive(t, *first.SizeBytes)
assert.Nil(t, second.SizeBytes)
Expand Down
2 changes: 1 addition & 1 deletion cmd/air/snapshot_cachekey.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ const snapshotPackagingVersion = "v1"
// with a git_archive key) and lets us invalidate it if the fingerprint scheme changes.
// computePlainTarKey also folds in the shared snapshotPackagingVersion, so a
// packaging-logic bump invalidates both modes' keys.
const plainTarKeyVersion = "plaintar-v2"
const plainTarKeyVersion = "plaintar-v3"

// computePlainTarKey returns a content-addressed key for a working-tree snapshot: the
// SHA-256 over every file's path, size, mtime, mode, and symlink target (sorted for
Expand Down
48 changes: 16 additions & 32 deletions cmd/air/snapshot_package.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ import (
"strings"

"github.com/databricks/cli/libs/tarpack"
"github.com/klauspost/pgzip"
)

// Tar builders ported from cli/utils/snapshot.py. The git-ref path snapshots a commit
Expand Down Expand Up @@ -50,46 +49,31 @@ func createGitArchiveSnapshot(ctx context.Context, git gitRepo, commitSHA, outpu
return nil
}

// createPlainTarball writes a gzipped tar of the given working-tree files to
// outputTarball. The archive preserves repoPath's directory name as the top-level
// entry. files come pre-resolved from snapshotFiles (.gitignore honored, .git and
// macOS AppleDouble excluded), so the caller can reuse the same listing to
// content-address the upload.
func createPlainTarball(ctx context.Context, repoPath, outputTarball string, files []snapshotFile) error {
// createPlainTarball writes the given working-tree files to outputTarball as a
// segmented tarball (see snapshot_segment.go) and returns its index. The archive
// preserves repoPath's directory name as the top-level entry. files come pre-resolved
// from snapshotFiles (.gitignore honored, .git and macOS AppleDouble excluded), so the
// caller can reuse the same listing to content-address the upload.
func createPlainTarball(ctx context.Context, repoPath, outputTarball string, files []snapshotFile) (segmentIndex, error) {
dirName := filepath.Base(repoPath)
entries, sizes := plainTarEntries(repoPath, files)
return writeSegmentedTarball(ctx, dirName, outputTarball, entries, sizes, segmentTargetBytes, segmentMaxMembers)
}

// plainTarEntries returns the tar entries for files under repoPath's directory name,
// with each entry's estimated tar size for balancing members.
func plainTarEntries(repoPath string, files []snapshotFile) ([]tarpack.Entry, []int64) {
dirName := filepath.Base(repoPath)
entries := make([]tarpack.Entry, len(files))
sizes := make([]int64, len(files))
for i, file := range files {
entries[i] = tarpack.Entry{
Name: filepath.ToSlash(filepath.Join(dirName, file.rel)),
Path: filepath.Join(repoPath, file.rel),
}
sizes[i] = file.tarSize()
}

out, err := os.Create(outputTarball)
if err != nil {
return fmt.Errorf("failed to create tarball: %w", err)
}

// Gzip in parallel with klauspost/pgzip rather than a single-threaded writer:
// compression dominates packaging time on a large tree, and pgzip spreads it
// across cores. tarpack writes the tar in Go, so there is no `tar` subprocess
// (and no Windows colon-in-path issue a `tar -f` argument would hit).
gz, err := pgzip.NewWriterLevel(out, pgzip.DefaultCompression)
if err != nil {
out.Close()
return err
}
if err := tarpack.Write(ctxWriter{ctx, gz}, entries); err != nil {
gz.Close()
out.Close()
return fmt.Errorf("failed to create plain tarball: %w", err)
}
if err := gz.Close(); err != nil {
out.Close()
return fmt.Errorf("failed to finalize gzip: %w", err)
}
return out.Close()
return entries, sizes
}

// ctxWriter fails writes once ctx is done, so a cancelled submission (for
Expand Down
12 changes: 6 additions & 6 deletions cmd/air/snapshot_package_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ func TestCreatePlainTarball(t *testing.T) {
out := filepath.Join(t.TempDir(), "snap.tar.gz")
files, err := snapshotFiles(ctx, repo, nil, false)
require.NoError(t, err)
require.NoError(t, createPlainTarball(ctx, repo, out, files))
writePlainTarball(t, ctx, repo, out, files)

dirName := filepath.Base(repo)
entries := tarballEntries(t, out)
Expand All @@ -155,7 +155,7 @@ func TestCreatePlainTarballStopsWhenCancelled(t *testing.T) {

ctx, cancel := context.WithCancel(t.Context())
cancel()
err = createPlainTarball(ctx, repo, filepath.Join(t.TempDir(), "snap.tar.gz"), files)
_, err = createPlainTarball(ctx, repo, filepath.Join(t.TempDir(), "snap.tar.gz"), files)
require.ErrorIs(t, err, context.Canceled)
}

Expand All @@ -169,7 +169,7 @@ func TestCreatePlainTarball_HonorsGitignore(t *testing.T) {
out := filepath.Join(t.TempDir(), "snap.tar.gz")
files, err := snapshotFiles(ctx, repo, nil, false)
require.NoError(t, err)
require.NoError(t, createPlainTarball(ctx, repo, out, files))
writePlainTarball(t, ctx, repo, out, files)

dirName := filepath.Base(repo)
entries := tarballEntries(t, out)
Expand All @@ -186,7 +186,7 @@ func TestCreatePlainTarball_IncludePaths(t *testing.T) {
out := filepath.Join(t.TempDir(), "snap.tar.gz")
files, err := snapshotFiles(ctx, repo, []string{"src"}, false)
require.NoError(t, err)
require.NoError(t, createPlainTarball(ctx, repo, out, files))
writePlainTarball(t, ctx, repo, out, files)

dirName := filepath.Base(repo)
entries := tarballEntries(t, out)
Expand All @@ -208,7 +208,7 @@ func TestCreatePlainTarball_HonorsNestedGitignoreAndNegation(t *testing.T) {
out := filepath.Join(t.TempDir(), "snap.tar.gz")
files, err := snapshotFiles(t.Context(), repo, nil, false)
require.NoError(t, err)
require.NoError(t, createPlainTarball(t.Context(), repo, out, files))
writePlainTarball(t, t.Context(), repo, out, files)

dirName := filepath.Base(repo)
entries := tarballEntries(t, out)
Expand All @@ -230,7 +230,7 @@ func TestCreatePlainTarball_SkipsDeletedTrackedFiles(t *testing.T) {
out := filepath.Join(t.TempDir(), "snap.tar.gz")
files, err := snapshotFiles(t.Context(), repo, nil, true)
require.NoError(t, err)
require.NoError(t, createPlainTarball(t.Context(), repo, out, files))
writePlainTarball(t, t.Context(), repo, out, files)

dirName := filepath.Base(repo)
entries := tarballEntries(t, out)
Expand Down
196 changes: 196 additions & 0 deletions cmd/air/snapshot_segment.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,196 @@
package aircmd

import (
"compress/gzip"
"context"
"fmt"
"io"
"os"
"path/filepath"
"runtime"

"github.com/databricks/cli/libs/tarpack"
"golang.org/x/sync/errgroup"
)

// A segmented snapshot is a .tar.gz made of several concatenated gzip members whose
// uncompressed payload is one tar archive. Plain `tar -xzf` reads it as a single
// stream, so launchers that predate the index keep working. The index sidecar
// (<tarball>.idx.json) lets a launcher decompress members in parallel.

const (
segmentFormat = "air-segmented-tar-gz"
segmentFormatVersion = 1
segmentIndexSuffix = ".idx.json"
segmentTargetBytes = 32 << 20
segmentMaxMembers = 16

tarBlockSize = 512
)

// segmentIndex describes the members of a segmented tarball. Members are contiguous
// and cover [0, Size).
type segmentIndex struct {
Format string `json:"format"`
Version int `json:"version"`
Component string `json:"component"`
Size int64 `json:"size"`
Members []segmentMember `json:"members"`
}

// segmentMember is the byte range of one gzip member in the tarball.
type segmentMember struct {
Offset int64 `json:"offset"`
Length int64 `json:"length"`
}

// tarSize estimates the uncompressed tar bytes the entry occupies (header plus padded
// data), matching what tarpack writes for regular files and symlinks. Other file types
// are skipped by tarpack and take no space.
func (f snapshotFile) tarSize() int64 {
mode := os.FileMode(f.mode)
switch {
case mode.IsRegular():
return tarBlockSize + (f.size+tarBlockSize-1)/tarBlockSize*tarBlockSize
case mode&os.ModeSymlink != 0:
return tarBlockSize
}
return 0
}

// planSegments returns the index of the first entry of each member, given each
// entry's tar size. Members are contiguous and balanced by bytes: the member count is
// ceil(total/targetBytes) capped at maxMembers, and an entry goes to the member
// containing its starting offset. There is always at least one member.
func planSegments(sizes []int64, targetBytes int64, maxMembers int) []int {
var total int64
for _, s := range sizes {
total += s
}
if total == 0 {
return []int{0}
}
n := max(1, min(maxMembers, int((total+targetBytes-1)/targetBytes)))

starts := []int{0}
var offset int64
current := 0
for i, s := range sizes {
member := int(offset * int64(n) / total)
if member != current {
starts = append(starts, i)
current = member
}
offset += s
}
return starts
}

// writeSegmentedTarball writes entries to outputTarball as independently compressed
// gzip members and returns the index of their byte ranges. sizes[i] is the estimated
// tar size of entries[i] and is used only for balancing.
func writeSegmentedTarball(ctx context.Context, component, outputTarball string, entries []tarpack.Entry, sizes []int64, targetBytes int64, maxMembers int) (segmentIndex, error) {
starts := planSegments(sizes, targetBytes, maxMembers)
return writeSegments(ctx, component, outputTarball, len(starts), func(ctx context.Context, i int, w io.Writer) error {
end := len(entries)
if i+1 < len(starts) {
end = starts[i+1]
}
return writeMemberTar(ctx, w, entries[starts[i]:end], i == len(starts)-1)
})
}

// writeSegments runs writeMember for n members, spooling each to a temp file with at most
// runtime.NumCPU() in flight, then appends the files to outputTarball in member order. The
// output is assembled only after every member succeeds, so a failed member never leaves a
// partial tarball behind.
func writeSegments(ctx context.Context, component, outputTarball string, n int, writeMember func(ctx context.Context, i int, w io.Writer) error) (segmentIndex, error) {
dir, err := os.MkdirTemp("", "air-snapshot-members-")
if err != nil {
return segmentIndex{}, fmt.Errorf("failed to create snapshot members dir: %w", err)
}
defer os.RemoveAll(dir)

paths := make([]string, n)
g, gctx := errgroup.WithContext(ctx)
g.SetLimit(runtime.NumCPU())
for i := range n {
paths[i] = filepath.Join(dir, fmt.Sprintf("member-%d", i))
g.Go(func() error {
return spoolFile(gctx, paths[i], func(ctx context.Context, w io.Writer) error {
return writeMember(ctx, i, w)
})
})
}
if err := g.Wait(); err != nil {
return segmentIndex{}, err
}
return concatMembers(component, outputTarball, paths)
}

// spoolFile creates path and fills it with the output of write.
func spoolFile(ctx context.Context, path string, write func(ctx context.Context, w io.Writer) error) error {
f, err := os.Create(path)
if err != nil {
return err
}
if err := write(ctx, f); err != nil {
f.Close()
return err
}
return f.Close()
}

// concatMembers appends the member files at paths, in order, to outputTarball and returns
// the index of their byte ranges.
func concatMembers(component, outputTarball string, paths []string) (segmentIndex, error) {
out, err := os.Create(outputTarball)
if err != nil {
return segmentIndex{}, fmt.Errorf("failed to create tarball: %w", err)
}
defer out.Close()

index := segmentIndex{Format: segmentFormat, Version: segmentFormatVersion, Component: component}
for _, p := range paths {
length, err := appendFile(out, p)
if err != nil {
return segmentIndex{}, fmt.Errorf("failed to append snapshot member: %w", err)
}
index.Members = append(index.Members, segmentMember{Offset: index.Size, Length: length})
index.Size += length
}
if err := out.Close(); err != nil {
return segmentIndex{}, fmt.Errorf("failed to finalize tarball: %w", err)
}
return index, nil
}

// writeMemberTar writes entries to w as one gzip member. Members other than the last
// omit the tar end-of-archive marker.
func writeMemberTar(ctx context.Context, w io.Writer, entries []tarpack.Entry, last bool) error {
gz, err := gzip.NewWriterLevel(ctxWriter{ctx, w}, gzip.DefaultCompression)
if err != nil {
return err
}
write := tarpack.WriteFragment
if last {
write = tarpack.Write
}
if err := write(gz, entries); err != nil {
return fmt.Errorf("failed to create plain tarball: %w", err)
}
if err := gz.Close(); err != nil {
return fmt.Errorf("failed to finalize gzip: %w", err)
}
return nil
}

// appendFile copies the file at path onto the end of w and returns the bytes copied.
func appendFile(w io.Writer, path string) (int64, error) {
f, err := os.Open(path)
if err != nil {
return 0, err
}
defer f.Close()
return io.Copy(w, f)
}
Loading
Loading