diff --git a/.nextchanges/air/segmented-snapshot.md b/.nextchanges/air/segmented-snapshot.md new file mode 100644 index 00000000000..32ad18367ac --- /dev/null +++ b/.nextchanges/air/segmented-snapshot.md @@ -0,0 +1 @@ +* `databricks air run` packages working-tree code snapshots as parallel gzip segments with an index, which shortens submission for large code directories. ([#7403](https://github.com/databricks/cli/pull/7403)) diff --git a/cmd/air/runsubmit_test.go b/cmd/air/runsubmit_test.go index 441a1c12d26..98bd7f9a4e0 100644 --- a/cmd/air/runsubmit_test.go +++ b/cmd/air/runsubmit_test.go @@ -832,11 +832,11 @@ func TestSubmitWorkloadPlainTarContentAddressed(t *testing.T) { // path) is what proves the skip: both submits resolve to the same content-addressed // name, so a set could not tell a skipped second submit from one that re-uploaded to // that same path. - snapshotUploads := 0 + var snapshotUploads atomic.Int32 server.Handle("POST", "/api/2.0/workspace-files/import-file/{path...}", func(req testserver.Request) any { p := req.Vars["path"] if strings.Contains(p, "/.air/repo_snapshots/") { - snapshotUploads++ + snapshotUploads.Add(1) } return req.Workspace.WorkspaceFilesImportFile(p, req.Body, req.URL.Query().Get("overwrite") == "true") }) @@ -864,8 +864,8 @@ code_source: sidecarStore, sidecarBase := testSidecarStore(t, w) first, err := uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase) require.NoError(t, err) - require.NotZero(t, snapshotUploads, "first submit should upload the tarball") - afterFirst := snapshotUploads + require.NotZero(t, snapshotUploads.Load(), "first submit should upload the tarball") + afterFirst := snapshotUploads.Load() second, err := uploadSnapshot(ctx, w, loaded.CodeSource.Snapshot, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase) require.NoError(t, err) @@ -877,7 +877,7 @@ code_source: // Same unchanged tree → identical remote path, and the second submit moved no bytes: // zero new import-file calls (a real skip), not a re-upload to the same name. assert.Equal(t, first.CodeSourcePath, second.CodeSourcePath) - assert.Equal(t, afterFirst, snapshotUploads, "unchanged plain_tar should skip the second upload") + assert.Equal(t, afterFirst, snapshotUploads.Load(), "unchanged plain_tar should skip the second upload") require.NotNil(t, first.SizeBytes) assert.Positive(t, *first.SizeBytes) assert.Nil(t, second.SizeBytes) diff --git a/cmd/air/snapshot_cachekey.go b/cmd/air/snapshot_cachekey.go index 5c2cf3f7b1d..7b278a3a8cf 100644 --- a/cmd/air/snapshot_cachekey.go +++ b/cmd/air/snapshot_cachekey.go @@ -20,7 +20,7 @@ const snapshotPackagingVersion = "v1" // with a git_archive key) and lets us invalidate it if the fingerprint scheme changes. // computePlainTarKey also folds in the shared snapshotPackagingVersion, so a // packaging-logic bump invalidates both modes' keys. -const plainTarKeyVersion = "plaintar-v2" +const plainTarKeyVersion = "plaintar-v3" // computePlainTarKey returns a content-addressed key for a working-tree snapshot: the // SHA-256 over every file's path, size, mtime, mode, and symlink target (sorted for diff --git a/cmd/air/snapshot_package.go b/cmd/air/snapshot_package.go index ca8a6942f5f..609c07950ad 100644 --- a/cmd/air/snapshot_package.go +++ b/cmd/air/snapshot_package.go @@ -12,7 +12,6 @@ import ( "strings" "github.com/databricks/cli/libs/tarpack" - "github.com/klauspost/pgzip" ) // Tar builders ported from cli/utils/snapshot.py. The git-ref path snapshots a commit @@ -50,46 +49,31 @@ func createGitArchiveSnapshot(ctx context.Context, git gitRepo, commitSHA, outpu return nil } -// createPlainTarball writes a gzipped tar of the given working-tree files to -// outputTarball. The archive preserves repoPath's directory name as the top-level -// entry. files come pre-resolved from snapshotFiles (.gitignore honored, .git and -// macOS AppleDouble excluded), so the caller can reuse the same listing to -// content-address the upload. -func createPlainTarball(ctx context.Context, repoPath, outputTarball string, files []snapshotFile) error { +// createPlainTarball writes the given working-tree files to outputTarball as a +// segmented tarball (see snapshot_segment.go) and returns its index. The archive +// preserves repoPath's directory name as the top-level entry. files come pre-resolved +// from snapshotFiles (.gitignore honored, .git and macOS AppleDouble excluded), so the +// caller can reuse the same listing to content-address the upload. +func createPlainTarball(ctx context.Context, repoPath, outputTarball string, files []snapshotFile) (segmentIndex, error) { dirName := filepath.Base(repoPath) + entries, sizes := plainTarEntries(repoPath, files) + return writeSegmentedTarball(ctx, dirName, outputTarball, entries, sizes, segmentTargetBytes, segmentMaxMembers) +} +// plainTarEntries returns the tar entries for files under repoPath's directory name, +// with each entry's estimated tar size for balancing members. +func plainTarEntries(repoPath string, files []snapshotFile) ([]tarpack.Entry, []int64) { + dirName := filepath.Base(repoPath) entries := make([]tarpack.Entry, len(files)) + sizes := make([]int64, len(files)) for i, file := range files { entries[i] = tarpack.Entry{ Name: filepath.ToSlash(filepath.Join(dirName, file.rel)), Path: filepath.Join(repoPath, file.rel), } + sizes[i] = file.tarSize() } - - out, err := os.Create(outputTarball) - if err != nil { - return fmt.Errorf("failed to create tarball: %w", err) - } - - // Gzip in parallel with klauspost/pgzip rather than a single-threaded writer: - // compression dominates packaging time on a large tree, and pgzip spreads it - // across cores. tarpack writes the tar in Go, so there is no `tar` subprocess - // (and no Windows colon-in-path issue a `tar -f` argument would hit). - gz, err := pgzip.NewWriterLevel(out, pgzip.DefaultCompression) - if err != nil { - out.Close() - return err - } - if err := tarpack.Write(ctxWriter{ctx, gz}, entries); err != nil { - gz.Close() - out.Close() - return fmt.Errorf("failed to create plain tarball: %w", err) - } - if err := gz.Close(); err != nil { - out.Close() - return fmt.Errorf("failed to finalize gzip: %w", err) - } - return out.Close() + return entries, sizes } // ctxWriter fails writes once ctx is done, so a cancelled submission (for diff --git a/cmd/air/snapshot_package_test.go b/cmd/air/snapshot_package_test.go index db42f634356..64a0aec5533 100644 --- a/cmd/air/snapshot_package_test.go +++ b/cmd/air/snapshot_package_test.go @@ -135,7 +135,7 @@ func TestCreatePlainTarball(t *testing.T) { out := filepath.Join(t.TempDir(), "snap.tar.gz") files, err := snapshotFiles(ctx, repo, nil, false) require.NoError(t, err) - require.NoError(t, createPlainTarball(ctx, repo, out, files)) + writePlainTarball(t, ctx, repo, out, files) dirName := filepath.Base(repo) entries := tarballEntries(t, out) @@ -155,7 +155,7 @@ func TestCreatePlainTarballStopsWhenCancelled(t *testing.T) { ctx, cancel := context.WithCancel(t.Context()) cancel() - err = createPlainTarball(ctx, repo, filepath.Join(t.TempDir(), "snap.tar.gz"), files) + _, err = createPlainTarball(ctx, repo, filepath.Join(t.TempDir(), "snap.tar.gz"), files) require.ErrorIs(t, err, context.Canceled) } @@ -169,7 +169,7 @@ func TestCreatePlainTarball_HonorsGitignore(t *testing.T) { out := filepath.Join(t.TempDir(), "snap.tar.gz") files, err := snapshotFiles(ctx, repo, nil, false) require.NoError(t, err) - require.NoError(t, createPlainTarball(ctx, repo, out, files)) + writePlainTarball(t, ctx, repo, out, files) dirName := filepath.Base(repo) entries := tarballEntries(t, out) @@ -186,7 +186,7 @@ func TestCreatePlainTarball_IncludePaths(t *testing.T) { out := filepath.Join(t.TempDir(), "snap.tar.gz") files, err := snapshotFiles(ctx, repo, []string{"src"}, false) require.NoError(t, err) - require.NoError(t, createPlainTarball(ctx, repo, out, files)) + writePlainTarball(t, ctx, repo, out, files) dirName := filepath.Base(repo) entries := tarballEntries(t, out) @@ -208,7 +208,7 @@ func TestCreatePlainTarball_HonorsNestedGitignoreAndNegation(t *testing.T) { out := filepath.Join(t.TempDir(), "snap.tar.gz") files, err := snapshotFiles(t.Context(), repo, nil, false) require.NoError(t, err) - require.NoError(t, createPlainTarball(t.Context(), repo, out, files)) + writePlainTarball(t, t.Context(), repo, out, files) dirName := filepath.Base(repo) entries := tarballEntries(t, out) @@ -230,7 +230,7 @@ func TestCreatePlainTarball_SkipsDeletedTrackedFiles(t *testing.T) { out := filepath.Join(t.TempDir(), "snap.tar.gz") files, err := snapshotFiles(t.Context(), repo, nil, true) require.NoError(t, err) - require.NoError(t, createPlainTarball(t.Context(), repo, out, files)) + writePlainTarball(t, t.Context(), repo, out, files) dirName := filepath.Base(repo) entries := tarballEntries(t, out) diff --git a/cmd/air/snapshot_segment.go b/cmd/air/snapshot_segment.go new file mode 100644 index 00000000000..6710fb6a6fa --- /dev/null +++ b/cmd/air/snapshot_segment.go @@ -0,0 +1,196 @@ +package aircmd + +import ( + "compress/gzip" + "context" + "fmt" + "io" + "os" + "path/filepath" + "runtime" + + "github.com/databricks/cli/libs/tarpack" + "golang.org/x/sync/errgroup" +) + +// A segmented snapshot is a .tar.gz made of several concatenated gzip members whose +// uncompressed payload is one tar archive. Plain `tar -xzf` reads it as a single +// stream, so launchers that predate the index keep working. The index sidecar +// (.idx.json) lets a launcher decompress members in parallel. + +const ( + segmentFormat = "air-segmented-tar-gz" + segmentFormatVersion = 1 + segmentIndexSuffix = ".idx.json" + segmentTargetBytes = 32 << 20 + segmentMaxMembers = 16 + + tarBlockSize = 512 +) + +// segmentIndex describes the members of a segmented tarball. Members are contiguous +// and cover [0, Size). +type segmentIndex struct { + Format string `json:"format"` + Version int `json:"version"` + Component string `json:"component"` + Size int64 `json:"size"` + Members []segmentMember `json:"members"` +} + +// segmentMember is the byte range of one gzip member in the tarball. +type segmentMember struct { + Offset int64 `json:"offset"` + Length int64 `json:"length"` +} + +// tarSize estimates the uncompressed tar bytes the entry occupies (header plus padded +// data), matching what tarpack writes for regular files and symlinks. Other file types +// are skipped by tarpack and take no space. +func (f snapshotFile) tarSize() int64 { + mode := os.FileMode(f.mode) + switch { + case mode.IsRegular(): + return tarBlockSize + (f.size+tarBlockSize-1)/tarBlockSize*tarBlockSize + case mode&os.ModeSymlink != 0: + return tarBlockSize + } + return 0 +} + +// planSegments returns the index of the first entry of each member, given each +// entry's tar size. Members are contiguous and balanced by bytes: the member count is +// ceil(total/targetBytes) capped at maxMembers, and an entry goes to the member +// containing its starting offset. There is always at least one member. +func planSegments(sizes []int64, targetBytes int64, maxMembers int) []int { + var total int64 + for _, s := range sizes { + total += s + } + if total == 0 { + return []int{0} + } + n := max(1, min(maxMembers, int((total+targetBytes-1)/targetBytes))) + + starts := []int{0} + var offset int64 + current := 0 + for i, s := range sizes { + member := int(offset * int64(n) / total) + if member != current { + starts = append(starts, i) + current = member + } + offset += s + } + return starts +} + +// writeSegmentedTarball writes entries to outputTarball as independently compressed +// gzip members and returns the index of their byte ranges. sizes[i] is the estimated +// tar size of entries[i] and is used only for balancing. +func writeSegmentedTarball(ctx context.Context, component, outputTarball string, entries []tarpack.Entry, sizes []int64, targetBytes int64, maxMembers int) (segmentIndex, error) { + starts := planSegments(sizes, targetBytes, maxMembers) + return writeSegments(ctx, component, outputTarball, len(starts), func(ctx context.Context, i int, w io.Writer) error { + end := len(entries) + if i+1 < len(starts) { + end = starts[i+1] + } + return writeMemberTar(ctx, w, entries[starts[i]:end], i == len(starts)-1) + }) +} + +// writeSegments runs writeMember for n members, spooling each to a temp file with at most +// runtime.NumCPU() in flight, then appends the files to outputTarball in member order. The +// output is assembled only after every member succeeds, so a failed member never leaves a +// partial tarball behind. +func writeSegments(ctx context.Context, component, outputTarball string, n int, writeMember func(ctx context.Context, i int, w io.Writer) error) (segmentIndex, error) { + dir, err := os.MkdirTemp("", "air-snapshot-members-") + if err != nil { + return segmentIndex{}, fmt.Errorf("failed to create snapshot members dir: %w", err) + } + defer os.RemoveAll(dir) + + paths := make([]string, n) + g, gctx := errgroup.WithContext(ctx) + g.SetLimit(runtime.NumCPU()) + for i := range n { + paths[i] = filepath.Join(dir, fmt.Sprintf("member-%d", i)) + g.Go(func() error { + return spoolFile(gctx, paths[i], func(ctx context.Context, w io.Writer) error { + return writeMember(ctx, i, w) + }) + }) + } + if err := g.Wait(); err != nil { + return segmentIndex{}, err + } + return concatMembers(component, outputTarball, paths) +} + +// spoolFile creates path and fills it with the output of write. +func spoolFile(ctx context.Context, path string, write func(ctx context.Context, w io.Writer) error) error { + f, err := os.Create(path) + if err != nil { + return err + } + if err := write(ctx, f); err != nil { + f.Close() + return err + } + return f.Close() +} + +// concatMembers appends the member files at paths, in order, to outputTarball and returns +// the index of their byte ranges. +func concatMembers(component, outputTarball string, paths []string) (segmentIndex, error) { + out, err := os.Create(outputTarball) + if err != nil { + return segmentIndex{}, fmt.Errorf("failed to create tarball: %w", err) + } + defer out.Close() + + index := segmentIndex{Format: segmentFormat, Version: segmentFormatVersion, Component: component} + for _, p := range paths { + length, err := appendFile(out, p) + if err != nil { + return segmentIndex{}, fmt.Errorf("failed to append snapshot member: %w", err) + } + index.Members = append(index.Members, segmentMember{Offset: index.Size, Length: length}) + index.Size += length + } + if err := out.Close(); err != nil { + return segmentIndex{}, fmt.Errorf("failed to finalize tarball: %w", err) + } + return index, nil +} + +// writeMemberTar writes entries to w as one gzip member. Members other than the last +// omit the tar end-of-archive marker. +func writeMemberTar(ctx context.Context, w io.Writer, entries []tarpack.Entry, last bool) error { + gz, err := gzip.NewWriterLevel(ctxWriter{ctx, w}, gzip.DefaultCompression) + if err != nil { + return err + } + write := tarpack.WriteFragment + if last { + write = tarpack.Write + } + if err := write(gz, entries); err != nil { + return fmt.Errorf("failed to create plain tarball: %w", err) + } + if err := gz.Close(); err != nil { + return fmt.Errorf("failed to finalize gzip: %w", err) + } + return nil +} + +// appendFile copies the file at path onto the end of w and returns the bytes copied. +func appendFile(w io.Writer, path string) (int64, error) { + f, err := os.Open(path) + if err != nil { + return 0, err + } + defer f.Close() + return io.Copy(w, f) +} diff --git a/cmd/air/snapshot_segment_test.go b/cmd/air/snapshot_segment_test.go new file mode 100644 index 00000000000..7465911cad1 --- /dev/null +++ b/cmd/air/snapshot_segment_test.go @@ -0,0 +1,196 @@ +package aircmd + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/databricks/cli/libs/tarpack" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// writePlainTarball packages files with createPlainTarball and fails the test on error. +func writePlainTarball(t *testing.T, ctx context.Context, repo, out string, files []snapshotFile) { + t.Helper() + _, err := createPlainTarball(ctx, repo, out, files) + require.NoError(t, err) +} + +// tarNames reads a tar stream to its end and returns the entry names. A clean EOF is +// accepted at an entry boundary, so it also reads a member fragment with no end marker. +func tarNames(t *testing.T, r io.Reader) []string { + t.Helper() + var names []string + tr := tar.NewReader(r) + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + return names + } + require.NoError(t, err) + names = append(names, hdr.Name) + _, err = io.Copy(io.Discard, tr) + require.NoError(t, err) + } +} + +// segmentFixture is a repo packaged as a multi-member segmented tarball. The small target +// forces several members. +type segmentFixture struct { + entries []tarpack.Entry + index segmentIndex + data []byte +} + +func newSegmentFixture(t *testing.T) segmentFixture { + t.Helper() + repo := filepath.Join(t.TempDir(), "code") + for i := range 30 { + writeRepoFile(t, repo, fmt.Sprintf("pkg%d/file%d.py", i%3, i), strings.Repeat(fmt.Sprintf("x = %d\n", i), 50*i+1)) + } + files, err := snapshotFiles(t.Context(), repo, nil, false) + require.NoError(t, err) + entries, sizes := plainTarEntries(repo, files) + + out := filepath.Join(t.TempDir(), "snap.tar.gz") + index, err := writeSegmentedTarball(t.Context(), "code", out, entries, sizes, 4096, 16) + require.NoError(t, err) + require.Greater(t, len(index.Members), 1) + data, err := os.ReadFile(out) + require.NoError(t, err) + return segmentFixture{entries: entries, index: index, data: data} +} + +func (f segmentFixture) names() []string { + var names []string + for _, e := range f.entries { + names = append(names, e.Name) + } + return names +} + +func TestPlanSegments(t *testing.T) { + tests := []struct { + name string + sizes []int64 + target int64 + maxMembers int + want []int + }{ + {name: "no entries is one member", sizes: nil, target: 100, maxMembers: 16, want: []int{0}}, + {name: "all zero-size entries is one member", sizes: []int64{0, 0}, target: 100, maxMembers: 16, want: []int{0}}, + {name: "under target is one member", sizes: []int64{40, 40}, target: 100, maxMembers: 16, want: []int{0}}, + {name: "balanced by bytes", sizes: []int64{50, 50, 50, 50}, target: 100, maxMembers: 16, want: []int{0, 2}}, + {name: "big entry is not split", sizes: []int64{300, 10}, target: 100, maxMembers: 16, want: []int{0, 1}}, + {name: "capped at max members", sizes: []int64{10, 10, 10, 10, 10, 10}, target: 10, maxMembers: 2, want: []int{0, 3}}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, planSegments(tc.sizes, tc.target, tc.maxMembers)) + }) + } +} + +// Old launchers read the whole .tar.gz as one gzip stream: it must yield exactly the +// expected entries, even with several members. +func TestSegmentedTarballReadsAsSingleStream(t *testing.T) { + f := newSegmentFixture(t) + gz, err := gzip.NewReader(bytes.NewReader(f.data)) + require.NoError(t, err) + assert.Equal(t, f.names(), tarNames(t, gz)) +} + +// Each member decodes on its own to a tar fragment, and the members in order are the full +// entry set. Only the last member carries the end-of-archive marker. +func TestSegmentedTarballMembersDecodeIndependently(t *testing.T) { + f := newSegmentFixture(t) + var union []string + for i, m := range f.index.Members { + member := f.data[m.Offset : m.Offset+m.Length] + gz, err := gzip.NewReader(bytes.NewReader(member)) + require.NoError(t, err) + payload, err := io.ReadAll(gz) + require.NoError(t, err) + endsWithMarker := bytes.HasSuffix(payload, make([]byte, 1024)) + assert.Equal(t, i == len(f.index.Members)-1, endsWithMarker, "member %d", i) + + gz, err = gzip.NewReader(bytes.NewReader(member)) + require.NoError(t, err) + union = append(union, tarNames(t, gz)...) + } + assert.Equal(t, f.names(), union) +} + +// The index matches the tarball: members are contiguous, cover [0, size) with size equal +// to the file size, and every entry sits under the component directory. +func TestSegmentedTarballIndexContract(t *testing.T) { + f := newSegmentFixture(t) + assert.Equal(t, segmentFormat, f.index.Format) + assert.Equal(t, segmentFormatVersion, f.index.Version) + assert.Equal(t, "code", f.index.Component) + assert.Equal(t, int64(len(f.data)), f.index.Size) + + var next int64 + for _, m := range f.index.Members { + assert.Equal(t, next, m.Offset) + assert.Positive(t, m.Length) + next += m.Length + } + assert.Equal(t, f.index.Size, next) + + for _, name := range f.names() { + assert.True(t, strings.HasPrefix(name, "code/"), name) + } +} + +// A small repo packages as one member, and the index covers the whole file. +func TestCreatePlainTarballSmallRepoIsOneMember(t *testing.T) { + repo := filepath.Join(t.TempDir(), "code") + writeRepoFile(t, repo, "a.py", "print()") + files, err := snapshotFiles(t.Context(), repo, nil, false) + require.NoError(t, err) + out := filepath.Join(t.TempDir(), "snap.tar.gz") + index, err := createPlainTarball(t.Context(), repo, out, files) + require.NoError(t, err) + + info, err := os.Stat(out) + require.NoError(t, err) + assert.Equal(t, []segmentMember{{Offset: 0, Length: info.Size()}}, index.Members) + assert.Equal(t, info.Size(), index.Size) +} + +// Same input yields byte-identical tarballs and indexes, which the content-addressed cache +// key depends on. +func TestCreatePlainTarballIsDeterministic(t *testing.T) { + repo := filepath.Join(t.TempDir(), "code") + for i := range 30 { + writeRepoFile(t, repo, fmt.Sprintf("f%d.py", i), strings.Repeat("x", 300*i+1)) + } + files, err := snapshotFiles(t.Context(), repo, nil, false) + require.NoError(t, err) + entries, sizes := plainTarEntries(repo, files) + + dir := t.TempDir() + write := func(name string) (segmentIndex, []byte) { + out := filepath.Join(dir, name) + index, err := writeSegmentedTarball(t.Context(), "code", out, entries, sizes, 4096, 16) + require.NoError(t, err) + data, err := os.ReadFile(out) + require.NoError(t, err) + return index, data + } + indexA, dataA := write("a.tar.gz") + indexB, dataB := write("b.tar.gz") + require.Greater(t, len(indexA.Members), 1) + assert.Equal(t, dataA, dataB) + assert.Equal(t, indexA, indexB) +} diff --git a/cmd/air/snapshot_upload.go b/cmd/air/snapshot_upload.go index 1d4e9a533b8..b22362457ab 100644 --- a/cmd/air/snapshot_upload.go +++ b/cmd/air/snapshot_upload.go @@ -3,12 +3,14 @@ package aircmd import ( "bytes" "context" + "encoding/json" "errors" "fmt" "io/fs" "os" "path" "path/filepath" + "slices" "strings" "time" @@ -16,6 +18,7 @@ import ( "github.com/databricks/cli/libs/filer" "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go" + "golang.org/x/sync/errgroup" ) // uploadProvenanceSidecars controls collection and upload of git_state.json @@ -137,13 +140,18 @@ func snapshotTarName(ctx context.Context, repoPath string, plan snapshotPlan) (s } // packageSnapshot writes the snapshot to tarball per the resolved plan: `git archive` -// of the pinned commit for git_archive, else a plain tar of the pre-listed working-tree -// files (nil for git_archive). -func packageSnapshot(ctx context.Context, repoPath string, plan snapshotPlan, files []snapshotFile, tarball string) error { +// of the pinned commit for git_archive, else a segmented plain tar of the pre-listed +// working-tree files. It returns the segment index for plain_tar and nil for +// git_archive, which has no sidecar yet. +func packageSnapshot(ctx context.Context, repoPath string, plan snapshotPlan, files []snapshotFile, tarball string) (*segmentIndex, error) { if plan.mode == modeGitArchive { - return createGitArchiveSnapshot(ctx, newGitRepo(repoPath), plan.commitSHA, tarball, filepath.Base(repoPath), plan.includePaths, plan.subtreePrefix) + return nil, createGitArchiveSnapshot(ctx, newGitRepo(repoPath), plan.commitSHA, tarball, filepath.Base(repoPath), plan.includePaths, plan.subtreePrefix) } - return createPlainTarball(ctx, repoPath, tarball, files) + index, err := createPlainTarball(ctx, repoPath, tarball, files) + if err != nil { + return nil, err + } + return &index, nil } // airArtifactInternalDir mirrors bundle/libraries.InternalDirName: CLI-uploaded @@ -172,11 +180,11 @@ func uploadSnapshotTarball(ctx context.Context, w *databricks.WorkspaceClient, r // upload the bytes now or reuse an object already in the store. remote := path.Join(uploadPath, tarName) - exists, err := snapshotExists(ctx, f, tarName) + cached, err := snapshotCached(ctx, f, tarName, plan.mode == modePlainTar) if err != nil { return result, err } - if exists { + if cached { log.Debugf(ctx, "snapshot upload skipped; reusing %s", remote) result.CodeSourcePath = remote result.PackagingDurationMs = new(int64(0)) @@ -192,7 +200,7 @@ func uploadSnapshotTarball(ctx context.Context, w *databricks.WorkspaceClient, r tarball := filepath.Join(tmp, tarName) packagingStart := time.Now() - err = packageSnapshot(ctx, repoPath, plan, files, tarball) + index, err := packageSnapshot(ctx, repoPath, plan, files, tarball) result.PackagingDurationMs = new(time.Since(packagingStart).Milliseconds()) if err != nil { return result, err @@ -212,15 +220,63 @@ func uploadSnapshotTarball(ctx context.Context, w *databricks.WorkspaceClient, r } cmdio.LogProgress(ctx, fmt.Sprintf("Uploading %s...", tarName)) uploadStart := time.Now() - err = f.Write(ctx, tarName, file, filer.OverwriteIfExists, filer.CreateParentDirectories) + // The sidecar and tarball are written concurrently. The job is only submitted once both + // succeed, and a later cache hit requires both (see snapshotCached). + g, gctx := errgroup.WithContext(ctx) + if index != nil { + g.Go(func() error { return uploadSegmentIndex(gctx, f, tarName, index) }) + } + g.Go(func() error { + if err := f.Write(gctx, tarName, file, filer.OverwriteIfExists, filer.CreateParentDirectories); err != nil { + return fmt.Errorf("failed to upload snapshot %s: %w", tarName, err) + } + return nil + }) + err = g.Wait() result.UploadDurationMs = new(time.Since(uploadStart).Milliseconds()) if err != nil { - return result, fmt.Errorf("failed to upload snapshot %s: %w", tarName, err) + return result, err } result.CodeSourcePath = remote return result, nil } +// snapshotCached reports whether the snapshot is already in the store. For an indexed +// (plain_tar) snapshot both the tarball and its index sidecar must exist: a half-written +// pair from an interrupted upload is a miss, and the pair is re-uploaded. +func snapshotCached(ctx context.Context, store filer.Filer, tarName string, indexed bool) (bool, error) { + names := []string{tarName} + if indexed { + names = append(names, tarName+segmentIndexSuffix) + } + exists := make([]bool, len(names)) + g, gctx := errgroup.WithContext(ctx) + for i, name := range names { + g.Go(func() error { + ok, err := snapshotExists(gctx, store, name) + exists[i] = ok + return err + }) + } + if err := g.Wait(); err != nil { + return false, err + } + return !slices.Contains(exists, false), nil +} + +// uploadSegmentIndex writes the index sidecar for tarName next to it in the store. +func uploadSegmentIndex(ctx context.Context, store filer.Filer, tarName string, index *segmentIndex) error { + data, err := json.Marshal(index) + if err != nil { + return fmt.Errorf("failed to encode snapshot index: %w", err) + } + err = store.Write(ctx, tarName+segmentIndexSuffix, bytes.NewReader(data), filer.OverwriteIfExists, filer.CreateParentDirectories) + if err != nil { + return fmt.Errorf("failed to upload snapshot index for %s: %w", tarName, err) + } + return nil +} + // snapshotUploadFiler returns a filer rooted at /.internal plus that // upload path. It routes /Volumes paths to a Files API filer and everything else to a // workspace files filer, matching bundle artifact-upload resolution without depending diff --git a/cmd/air/snapshot_upload_test.go b/cmd/air/snapshot_upload_test.go new file mode 100644 index 00000000000..3c6c7af64f8 --- /dev/null +++ b/cmd/air/snapshot_upload_test.go @@ -0,0 +1,143 @@ +package aircmd + +import ( + "encoding/json" + "path" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/filer" + "github.com/databricks/cli/libs/testserver" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// uploadRecorder records which snapshot objects the fake workspace receives. The import +// handler runs concurrently with the sidecar and tarball writes, so it is mutex-guarded. +type uploadRecorder struct { + mu sync.Mutex + written map[string]bool + index segmentIndex +} + +// reset forgets prior writes so a phase can assert on its own uploads. +func (r *uploadRecorder) reset() { + r.mu.Lock() + defer r.mu.Unlock() + r.written = map[string]bool{} +} + +// names returns the snapshot object names written since the last reset. +func (r *uploadRecorder) names() map[string]bool { + r.mu.Lock() + defer r.mu.Unlock() + out := map[string]bool{} + for k := range r.written { + out[k] = true + } + return out +} + +func (r *uploadRecorder) record(name string, body []byte) { + r.mu.Lock() + defer r.mu.Unlock() + r.written[name] = true + if strings.HasSuffix(name, segmentIndexSuffix) { + _ = json.Unmarshal(body, &r.index) + } +} + +// newUploadTestWorkspace returns a fake workspace that records snapshot object writes. +func newUploadTestWorkspace(t *testing.T) (*databricks.WorkspaceClient, *uploadRecorder) { + t.Helper() + server := testserver.New(t) + t.Cleanup(server.Close) + server.Handle("POST", "/api/2.2/jobs/runs/submit", func(req testserver.Request) any { + return jobs.SubmitRunResponse{RunId: 555} + }) + rec := &uploadRecorder{written: map[string]bool{}} + server.Handle("POST", "/api/2.0/workspace-files/import-file/{path...}", func(req testserver.Request) any { + p := req.Vars["path"] + if strings.Contains(p, "/.air/repo_snapshots/") { + rec.record(path.Base(p), req.Body) + } + return req.Workspace.WorkspaceFilesImportFile(p, req.Body, req.URL.Query().Get("overwrite") == "true") + }) + stubValidateConfig(server) + testserver.AddDefaultHandlers(server) + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "token"}) + require.NoError(t, err) + return w, rec +} + +// A plain-tar snapshot writes its index sidecar and tarball, and a later submit of the +// same tree is a cache hit that writes nothing. +func TestUploadSnapshotPlainTarWritesIndexAndTarball(t *testing.T) { + w, rec := newUploadTestWorkspace(t) + repo := filepath.Join(t.TempDir(), "src") + writeRepoFile(t, repo, "train.py", "print()") + snap, cfgPath := loadSnapshotConfig(t, repo) + + ctx := cmdio.MockDiscard(t.Context()) + sidecarStore, sidecarBase := testSidecarStore(t, w) + first, err := uploadSnapshot(ctx, w, snap, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase) + require.NoError(t, err) + + tarName := path.Base(first.CodeSourcePath) + assert.Equal(t, map[string]bool{tarName: true, tarName + segmentIndexSuffix: true}, rec.names()) + assert.Equal(t, segmentFormat, rec.index.Format) + assert.Equal(t, "src", rec.index.Component) + require.NotNil(t, first.SizeBytes) + assert.Equal(t, *first.SizeBytes, rec.index.Size) + assert.Len(t, rec.index.Members, 1) + + rec.reset() + second, err := uploadSnapshot(ctx, w, snap, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase) + require.NoError(t, err) + assert.Equal(t, first.CodeSourcePath, second.CodeSourcePath) + assert.Empty(t, rec.names(), "cache hit should write neither the index nor the tarball") +} + +// A tarball without its index sidecar is a cache miss: both objects are written again. +func TestUploadSnapshotPlainTarMissingSidecarReuploadsBoth(t *testing.T) { + w, rec := newUploadTestWorkspace(t) + repo := filepath.Join(t.TempDir(), "src") + writeRepoFile(t, repo, "train.py", "print()") + snap, cfgPath := loadSnapshotConfig(t, repo) + + ctx := cmdio.MockDiscard(t.Context()) + sidecarStore, sidecarBase := testSidecarStore(t, w) + first, err := uploadSnapshot(ctx, w, snap, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase) + require.NoError(t, err) + tarName := path.Base(first.CodeSourcePath) + + uploadDir, err := filer.NewWorkspaceFilesClient(w, testSnapshotArtifactPath+"/.internal") + require.NoError(t, err) + require.NoError(t, uploadDir.Delete(ctx, tarName+segmentIndexSuffix)) + + rec.reset() + second, err := uploadSnapshot(ctx, w, snap, cfgPath, testSnapshotArtifactPath, sidecarStore, sidecarBase) + require.NoError(t, err) + assert.Equal(t, first.CodeSourcePath, second.CodeSourcePath) + assert.Equal(t, map[string]bool{tarName: true, tarName + segmentIndexSuffix: true}, rec.names()) +} + +// loadSnapshotConfig writes a run config whose code_source is repo and returns its snapshot +// config and path. +func loadSnapshotConfig(t *testing.T, repo string) (*snapshotSourceConfig, string) { + t.Helper() + cfgPath := writeConfigFile(t, "run.yaml", minimalConfig+` +code_source: + type: snapshot + snapshot: + root_path: `+repo+` +`) + loaded, err := loadRunConfig(cfgPath) + require.NoError(t, err) + return loaded.CodeSource.Snapshot, cfgPath +} diff --git a/libs/tarpack/tarpack.go b/libs/tarpack/tarpack.go index 963988c0858..19de98d3248 100644 --- a/libs/tarpack/tarpack.go +++ b/libs/tarpack/tarpack.go @@ -29,12 +29,32 @@ type Entry struct { // matches what `tar` stored for the same file. func Write(w io.Writer, entries []Entry) error { tw := tar.NewWriter(w) + if err := writeEntries(tw, entries); err != nil { + return err + } + return tw.Close() +} + +// WriteFragment is Write without the end-of-archive marker. Concatenating +// fragments and then one Write-finalized archive yields a single tar stream, which +// lets callers split an archive into independently compressed parts. Each fragment +// ends on an entry boundary, so it can be read alone as a tar stream without its +// end marker. +func WriteFragment(w io.Writer, entries []Entry) error { + tw := tar.NewWriter(w) + if err := writeEntries(tw, entries); err != nil { + return err + } + return tw.Flush() +} + +func writeEntries(tw *tar.Writer, entries []Entry) error { for _, e := range entries { if err := writeEntry(tw, e); err != nil { return err } } - return tw.Close() + return nil } func writeEntry(tw *tar.Writer, e Entry) error { diff --git a/libs/tarpack/tarpack_test.go b/libs/tarpack/tarpack_test.go index 3841e779ac0..a3ccebd5989 100644 --- a/libs/tarpack/tarpack_test.go +++ b/libs/tarpack/tarpack_test.go @@ -96,6 +96,25 @@ func TestWriteSymlink(t *testing.T) { assert.Empty(t, got["pkg/link"].content) } +func TestWriteFragmentOmitsEndMarker(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "a.txt"), []byte("alpha"), 0o644)) + entries := []Entry{{Name: "pkg/a.txt", Path: filepath.Join(dir, "a.txt")}} + + var full, fragment bytes.Buffer + require.NoError(t, Write(&full, entries)) + require.NoError(t, WriteFragment(&fragment, entries)) + + // The fragment is the full archive minus the two zero end blocks. + assert.Equal(t, full.Bytes()[:fragment.Len()], fragment.Bytes()) + assert.Equal(t, 1024, full.Len()-fragment.Len()) + assert.Equal(t, make([]byte, 1024), full.Bytes()[fragment.Len():]) + + // Read as a standalone stream, the fragment yields its entry and then a clean EOF. + got := readTar(t, fragment.Bytes()) + assert.Equal(t, "alpha", got["pkg/a.txt"].content) +} + func TestWriteSkipsIrregularAndMissingIsError(t *testing.T) { dir := t.TempDir()