diff --git a/.nextchanges/air/git-archive-pgzip.md b/.nextchanges/air/git-archive-pgzip.md new file mode 100644 index 00000000000..f4a9820e728 --- /dev/null +++ b/.nextchanges/air/git-archive-pgzip.md @@ -0,0 +1 @@ +* Speed up `databricks air run` for git-pinned `code_source` snapshots by compressing the archive in parallel, which substantially shortens packaging on large repositories. ([#7402](https://github.com/databricks/cli/pull/7402)) diff --git a/.nextchanges/air/git-segmented-snapshot.md b/.nextchanges/air/git-segmented-snapshot.md new file mode 100644 index 00000000000..af2a7b255a8 --- /dev/null +++ b/.nextchanges/air/git-segmented-snapshot.md @@ -0,0 +1 @@ +* `databricks air run` with a pinned git commit packages the snapshot as parallel gzip segments with an index, which shortens submission for large repositories. ([#7404](https://github.com/databricks/cli/pull/7404)) diff --git a/cmd/air/runsubmit_test.go b/cmd/air/runsubmit_test.go index 98bd7f9a4e0..e6eb46e4491 100644 --- a/cmd/air/runsubmit_test.go +++ b/cmd/air/runsubmit_test.go @@ -904,11 +904,14 @@ func TestSubmitWorkloadGitArchiveCaching(t *testing.T) { // persistence so the second submit's cache-existence Stat sees the first upload. // Dedupe by path: the filer mkdirs-and-retries the import on a missing // parent dir, so one logical upload can hit this route more than once. + var mu sync.Mutex uploaded := map[string]bool{} server.Handle("POST", "/api/2.0/workspace-files/import-file/{path...}", func(req testserver.Request) any { p := req.Vars["path"] if strings.Contains(p, "/.air/repo_snapshots/") { + mu.Lock() uploaded[p] = true + mu.Unlock() } return req.Workspace.WorkspaceFilesImportFile(p, req.Body, req.URL.Query().Get("overwrite") == "true") }) @@ -943,7 +946,7 @@ code_source: // Same pinned commit → identical content-addressed remote path, uploaded once // (the second submit is a cache hit and moves no bytes). assert.Equal(t, first.CodeSourcePath, second.CodeSourcePath) - assert.Len(t, uploaded, 1, "git_archive cache hit should skip the second upload") + assert.Len(t, uploaded, 2, "git_archive cache hit should skip the second upload: tarball and index sidecar") require.NotNil(t, first.SizeBytes) assert.Positive(t, *first.SizeBytes) assert.Nil(t, second.SizeBytes) diff --git a/cmd/air/snapshot_cachekey.go b/cmd/air/snapshot_cachekey.go index 7b278a3a8cf..b130b17b56d 100644 --- a/cmd/air/snapshot_cachekey.go +++ b/cmd/air/snapshot_cachekey.go @@ -14,7 +14,7 @@ import ( ) // snapshotPackagingVersion is bumped when packaging logic changes in a way that invalidates existing caches -const snapshotPackagingVersion = "v1" +const snapshotPackagingVersion = "v2" // plainTarKeyVersion namespaces the plain_tar working-tree key (so it can never collide // with a git_archive key) and lets us invalidate it if the fingerprint scheme changes. diff --git a/cmd/air/snapshot_git.go b/cmd/air/snapshot_git.go index 6172ccec9c6..69972dd4d2b 100644 --- a/cmd/air/snapshot_git.go +++ b/cmd/air/snapshot_git.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "os/exec" "strings" "time" @@ -35,20 +36,30 @@ func (g gitRepo) run(ctx context.Context, args ...string) (string, error) { // runBytes is run returning raw stdout bytes, for the dirty-diff capture which needs // exact bytes and a size measurement. func (g gitRepo) runBytes(ctx context.Context, args ...string) ([]byte, error) { + var stdout bytes.Buffer + if err := g.runTo(ctx, &stdout, args...); err != nil { + return nil, err + } + return stdout.Bytes(), nil +} + +// runTo is run streaming stdout into w as git produces it, for output too large to +// buffer such as `git archive`. +func (g gitRepo) runTo(ctx context.Context, w io.Writer, args ...string) error { full := append([]string{"-C", g.path}, args...) cmd := exec.CommandContext(ctx, "git", full...) - var stdout, stderr bytes.Buffer - cmd.Stdout = &stdout + var stderr bytes.Buffer + cmd.Stdout = w cmd.Stderr = &stderr if err := cmd.Run(); err != nil { msg := strings.TrimSpace(stderr.String()) if msg == "" { - return nil, fmt.Errorf("git %s: %w", strings.Join(args, " "), err) + return fmt.Errorf("git %s: %w", strings.Join(args, " "), err) } - return nil, fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, msg) + return fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, msg) } - return stdout.Bytes(), nil + return nil } // isRepository reports whether the path is inside a git work tree. Using diff --git a/cmd/air/snapshot_git_segment_test.go b/cmd/air/snapshot_git_segment_test.go new file mode 100644 index 00000000000..e6955eeab8f --- /dev/null +++ b/cmd/air/snapshot_git_segment_test.go @@ -0,0 +1,105 @@ +package aircmd + +import ( + "bytes" + "compress/gzip" + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// mustCreateGitArchiveSnapshot packages commitSHA with createGitArchiveSnapshot and fails the +// test on error. +func mustCreateGitArchiveSnapshot(t *testing.T, ctx context.Context, git gitRepo, commitSHA, outputTarball, directoryName string, includePaths []string, subtreePrefix string) { + t.Helper() + _, err := createGitArchiveSnapshot(ctx, git, commitSHA, outputTarball, directoryName, includePaths, subtreePrefix) + require.NoError(t, err) +} + +// commitFiles commits n files of size bytes each, spread over three directories. +func commitFiles(t *testing.T, n, size int) (string, string) { + t.Helper() + repo := newTestRepo(t) + for i := range n { + writeRepoFile(t, repo, fmt.Sprintf("pkg%d/file%d.py", i%3, i), strings.Repeat("x", size+i)) + } + return repo, commitAll(t, repo, "init") +} + +// A multi-member git_archive snapshot reads as one stream with pax_global_header first. Its +// members are contiguous, decode independently to the same entries, and cover the file. +func TestCreateGitArchiveSnapshotSegmented(t *testing.T) { + repo, sha := commitFiles(t, 30, 200) + dirName := filepath.Base(repo) + + out := filepath.Join(t.TempDir(), "snap.tar.gz") + index, err := createGitArchiveSegments(t.Context(), newGitRepo(repo), sha, out, dirName, nil, "", 4096) + require.NoError(t, err) + require.Greater(t, len(index.Members), 1) + assert.Equal(t, dirName, index.Component) + + data, err := os.ReadFile(out) + require.NoError(t, err) + assert.Equal(t, int64(len(data)), index.Size) + + gz, err := gzip.NewReader(bytes.NewReader(data)) + require.NoError(t, err) + single := tarNames(t, gz) + require.NotEmpty(t, single) + assert.Equal(t, "pax_global_header", single[0]) + + var union []string + var next int64 + for i, m := range index.Members { + assert.Equal(t, next, m.Offset, "member %d not contiguous", i) + next += m.Length + member, err := gzip.NewReader(bytes.NewReader(data[m.Offset : m.Offset+m.Length])) + require.NoError(t, err) + names := tarNames(t, member) + if i == 0 { + require.NotEmpty(t, names) + assert.Equal(t, "pax_global_header", names[0]) + } + union = append(union, names...) + } + assert.Equal(t, index.Size, next) + assert.Equal(t, single, union) +} + +// An archive larger than maxMembers times the minimum target still splits into balanced +// members, instead of the last member absorbing the rest. Balance is checked on uncompressed +// tar bytes, which is what the launcher parallelizes over. +func TestCreateGitArchiveSnapshotBalancesLargeArchive(t *testing.T) { + const minTarget = 4096 + // 64 files of about 4 KiB: about 256 KiB of blobs, well over 16 times the 4 KiB minimum. + repo, sha := commitFiles(t, 64, 4000) + out := filepath.Join(t.TempDir(), "snap.tar.gz") + index, err := createGitArchiveSegments(t.Context(), newGitRepo(repo), sha, out, filepath.Base(repo), nil, "", minTarget) + require.NoError(t, err) + require.LessOrEqual(t, len(index.Members), segmentMaxMembers) + require.Greater(t, len(index.Members), 1) + + data, err := os.ReadFile(out) + require.NoError(t, err) + var sizes []int64 + var total int64 + for _, m := range index.Members { + gz, err := gzip.NewReader(bytes.NewReader(data[m.Offset : m.Offset+m.Length])) + require.NoError(t, err) + n, err := io.Copy(io.Discard, gz) + require.NoError(t, err) + sizes = append(sizes, n) + total += n + } + average := total / int64(len(sizes)) + for i, n := range sizes { + assert.LessOrEqual(t, n, 2*average, "member %d holds %d of %d uncompressed bytes", i, n, total) + } +} diff --git a/cmd/air/snapshot_package.go b/cmd/air/snapshot_package.go index 609c07950ad..c86ddbcc58d 100644 --- a/cmd/air/snapshot_package.go +++ b/cmd/air/snapshot_package.go @@ -9,6 +9,7 @@ import ( "os" "os/exec" "path/filepath" + "strconv" "strings" "github.com/databricks/cli/libs/tarpack" @@ -20,33 +21,104 @@ import ( // load-bearing — the remote entry_script extracts to /databricks/code_source/ — so // both forms prefix every entry with it. -// createGitArchiveSnapshot writes a gzipped tar of commitSHA to outputTarball via -// `git archive`, with every entry prefixed by directoryName/. When subtreePrefix is -// set, only that repository subtree is archived and includePaths are relative to it. -func createGitArchiveSnapshot(ctx context.Context, git gitRepo, commitSHA, outputTarball, directoryName string, includePaths []string, subtreePrefix string) error { +// createGitArchiveSnapshot writes commitSHA to outputTarball as a segmented tarball (see +// snapshot_segment.go) and returns its index. Every entry is prefixed by directoryName/. +// The tar is streamed from `git archive --format=tar`, since git's own tar.gz format +// compresses on one thread. When subtreePrefix is set, only that repository subtree is +// archived and includePaths are relative to it. +func createGitArchiveSnapshot(ctx context.Context, git gitRepo, commitSHA, outputTarball, directoryName string, includePaths []string, subtreePrefix string) (segmentIndex, error) { + return createGitArchiveSegments(ctx, git, commitSHA, outputTarball, directoryName, includePaths, subtreePrefix, segmentTargetBytes) +} + +// createGitArchiveSegments is createGitArchiveSnapshot with a minimum member size, so tests +// can force several members. Members are sized from the archive's blob total, so an +// archive larger than segmentMaxMembers times the minimum still splits evenly. +func createGitArchiveSegments(ctx context.Context, git gitRepo, commitSHA, outputTarball, directoryName string, includePaths []string, subtreePrefix string, minTargetBytes int64) (segmentIndex, error) { treeish := commitSHA archiveGit := git if subtreePrefix != "" { repoRoot, err := git.repositoryRoot(ctx) if err != nil { - return err + return segmentIndex{}, err } archiveGit = newGitRepo(repoRoot) treeish = commitSHA + ":" + subtreePrefix } + total, err := archiveGit.archiveTarSize(ctx, treeish, includePaths) + if err != nil { + return segmentIndex{}, err + } + targetBytes := max(minTargetBytes, (total+segmentMaxMembers-1)/segmentMaxMembers) + args := []string{ "archive", - "--format=tar.gz", + "--format=tar", "--prefix=" + directoryName + "/", - "-o", outputTarball, treeish, } args = append(args, includePaths...) - if _, err := archiveGit.run(ctx, args...); err != nil { - return fmt.Errorf("failed to create git archive: %w", err) + return streamGitArchive(ctx, archiveGit, args, outputTarball, directoryName, targetBytes, segmentMaxMembers) +} + +// archiveTarSize estimates the uncompressed tar bytes `git archive` writes for the blobs of +// treeish under includePaths, from `git ls-tree` sizes: a header block plus padded data per +// blob. Submodule entries have no blob size and take no space in the archive. +func (g gitRepo) archiveTarSize(ctx context.Context, treeish string, includePaths []string) (int64, error) { + args := []string{"ls-tree", "-r", "-l", "-z", treeish} + if len(includePaths) > 0 { + args = append(args, "--") + args = append(args, includePaths...) + } + out, err := g.runBytes(ctx, args...) + if err != nil { + return 0, fmt.Errorf("failed to create git archive: %w", err) + } + + var total int64 + for record := range bytes.SplitSeq(out, []byte{0}) { + // Each record is " \t". + meta, _, found := bytes.Cut(record, []byte{'\t'}) + if !found { + continue + } + fields := strings.Fields(string(meta)) + if len(fields) != 4 || fields[3] == "-" { + continue + } + size, err := strconv.ParseInt(fields[3], 10, 64) + if err != nil { + return 0, fmt.Errorf("failed to create git archive: %w", err) + } + total += tarBlockSize + (size+tarBlockSize-1)/tarBlockSize*tarBlockSize + } + return total, nil +} + +// streamGitArchive runs git with args and writes its tar output to outputTarball as segmented +// members. +func streamGitArchive(ctx context.Context, git gitRepo, args []string, outputTarball, component string, targetBytes int64, maxMembers int) (segmentIndex, error) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + pr, pw := io.Pipe() + gitDone := make(chan error, 1) + go func() { + err := git.runTo(ctx, pw, args...) + pw.CloseWithError(err) + gitDone <- err + }() + index, err := writeTarSegments(ctx, component, outputTarball, pr, targetBytes, maxMembers) + // git pads the archive past the end-of-archive marker, so drain it before closing the reader. + if err == nil { + _, err = io.Copy(io.Discard, pr) + } + // Closing the read side makes git's next write fail, so git exits if we stopped early. + pr.CloseWithError(err) + // git's own failure is the root cause when the tar side also failed. + if gitErr := <-gitDone; gitErr != nil { + return segmentIndex{}, fmt.Errorf("failed to create git archive: %w", gitErr) } - return nil + return index, err } // createPlainTarball writes the given working-tree files to outputTarball as a diff --git a/cmd/air/snapshot_package_test.go b/cmd/air/snapshot_package_test.go index 64a0aec5533..cef3d0f97ac 100644 --- a/cmd/air/snapshot_package_test.go +++ b/cmd/air/snapshot_package_test.go @@ -47,7 +47,7 @@ func TestCreateGitArchiveSnapshot(t *testing.T) { out := filepath.Join(t.TempDir(), "snap.tar.gz") dirName := filepath.Base(repo) - require.NoError(t, createGitArchiveSnapshot(ctx, newGitRepo(repo), sha, out, dirName, nil, "")) + mustCreateGitArchiveSnapshot(t, ctx, newGitRepo(repo), sha, out, dirName, nil, "") entries := tarballEntries(t, out) // Every real entry is prefixed with the directory name. git archive also emits a @@ -71,7 +71,7 @@ func TestCreateGitArchiveSnapshot_IncludePaths(t *testing.T) { out := filepath.Join(t.TempDir(), "snap.tar.gz") dirName := filepath.Base(repo) - require.NoError(t, createGitArchiveSnapshot(ctx, newGitRepo(repo), sha, out, dirName, []string{"src"}, "")) + mustCreateGitArchiveSnapshot(t, ctx, newGitRepo(repo), sha, out, dirName, []string{"src"}, "") entries := tarballEntries(t, out) assert.Contains(t, entries, dirName+"/src/model.py") @@ -91,7 +91,7 @@ func TestCreateGitArchiveSnapshot_SubdirectoryRootPath(t *testing.T) { require.NoError(t, err) out := filepath.Join(t.TempDir(), "snap.tar.gz") - require.NoError(t, createGitArchiveSnapshot(ctx, newGitRepo(rootPath), sha, out, "subpkg", nil, prefix)) + mustCreateGitArchiveSnapshot(t, ctx, newGitRepo(rootPath), sha, out, "subpkg", nil, prefix) entries := tarballEntries(t, out) assert.Contains(t, entries, "subpkg/train.py") @@ -115,7 +115,7 @@ func TestCreateGitArchiveSnapshot_SubdirectoryRootPathWithIncludePaths(t *testin require.NoError(t, err) out := filepath.Join(t.TempDir(), "snap.tar.gz") - require.NoError(t, createGitArchiveSnapshot(ctx, newGitRepo(rootPath), sha, out, "subpkg", []string{"src"}, prefix)) + mustCreateGitArchiveSnapshot(t, ctx, newGitRepo(rootPath), sha, out, "subpkg", []string{"src"}, prefix) entries := tarballEntries(t, out) assert.Contains(t, entries, "subpkg/src/model.py") @@ -123,6 +123,29 @@ func TestCreateGitArchiveSnapshot_SubdirectoryRootPathWithIncludePaths(t *testin assert.NotContains(t, entries, "subpkg/configs/train.yaml") } +func TestCreateGitArchiveSnapshot_GitFailureSurfacesStderr(t *testing.T) { + ctx := t.Context() + repo := newTestRepo(t) + writeRepoFile(t, repo, "a.txt", "1") + commitAll(t, repo, "init") + + out := filepath.Join(t.TempDir(), "snap.tar.gz") + _, err := createGitArchiveSnapshot(ctx, newGitRepo(repo), "0000000000000000000000000000000000000000", out, "repo", nil, "") + require.ErrorContains(t, err, "failed to create git archive") + assert.ErrorContains(t, err, "fatal") +} + +func TestCreateGitArchiveSnapshotStopsWhenCancelled(t *testing.T) { + repo := newTestRepo(t) + writeRepoFile(t, repo, "a.txt", "1") + sha := commitAll(t, repo, "init") + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + _, err := createGitArchiveSnapshot(ctx, newGitRepo(repo), sha, filepath.Join(t.TempDir(), "snap.tar.gz"), "repo", nil, "") + require.ErrorIs(t, err, context.Canceled) +} + func TestCreatePlainTarball(t *testing.T) { ctx := t.Context() repo := t.TempDir() diff --git a/cmd/air/snapshot_segment.go b/cmd/air/snapshot_segment.go index 6710fb6a6fa..4f1b95414c4 100644 --- a/cmd/air/snapshot_segment.go +++ b/cmd/air/snapshot_segment.go @@ -1,8 +1,10 @@ package aircmd import ( + "archive/tar" "compress/gzip" "context" + "errors" "fmt" "io" "os" @@ -185,6 +187,147 @@ func writeMemberTar(ctx context.Context, w io.Writer, entries []tarpack.Entry, l return nil } +// writeTarSegments streams the tar archive in tarStream to outputTarball as gzip members. The +// archive size is not known up front, so the caller passes targetBytes, the size at which a +// member closes at the next entry boundary. Once maxMembers members are open, the last one +// takes the rest. Each member is first written as raw tar to a temp file, so the stream is +// never blocked on compression. Members then compress concurrently, at most runtime.NumCPU() +// at a time, and the output is assembled only after all of them succeed. +func writeTarSegments(ctx context.Context, component, outputTarball string, tarStream io.Reader, targetBytes int64, maxMembers int) (segmentIndex, error) { + dir, err := os.MkdirTemp("", "air-snapshot-members-") + if err != nil { + return segmentIndex{}, fmt.Errorf("failed to create snapshot members dir: %w", err) + } + defer os.RemoveAll(dir) + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + g, gctx := errgroup.WithContext(ctx) + spool := &segmentSpool{dir: dir, g: g, slots: make(chan struct{}, runtime.NumCPU())} + streamErr := spool.stream(gctx, tarStream, targetBytes, maxMembers) + if streamErr != nil { + // The stream error is the root cause. Waiting only lets the compressors stop. + cancel() + _ = g.Wait() + return segmentIndex{}, streamErr + } + if err := g.Wait(); err != nil { + return segmentIndex{}, err + } + return concatMembers(component, outputTarball, spool.members) +} + +// segmentSpool writes members as raw tar files and compresses each one in the background. +type segmentSpool struct { + dir string + g *errgroup.Group + slots chan struct{} + members []string // compressed member paths, in order +} + +// rawMember is a member being written as raw tar, counting its bytes. +type rawMember struct { + raw *os.File + rawPath string + memberPath string + tar *tar.Writer + bytes int64 +} + +func (m *rawMember) Write(p []byte) (int, error) { + n, err := m.raw.Write(p) + m.bytes += int64(n) + return n, err +} + +// stream copies the entries of tarStream into members, closing a member at an entry boundary +// once it holds targetBytes, and never opening more than maxMembers. +func (s *segmentSpool) stream(ctx context.Context, tarStream io.Reader, targetBytes int64, maxMembers int) error { + cur, err := s.open() + if err != nil { + return err + } + tr := tar.NewReader(tarStream) + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return fmt.Errorf("failed to read git archive: %w", err) + } + if cur.bytes >= targetBytes && len(s.members) < maxMembers { + if err := s.close(ctx, cur, false); err != nil { + return err + } + if cur, err = s.open(); err != nil { + return err + } + } + if err := cur.tar.WriteHeader(hdr); err != nil { + return fmt.Errorf("failed to write snapshot entry %s: %w", hdr.Name, err) + } + if _, err := io.Copy(cur.tar, tr); err != nil { + return fmt.Errorf("failed to write snapshot entry %s: %w", hdr.Name, err) + } + } + return s.close(ctx, cur, true) +} + +// open starts the next member as a raw tar file. +func (s *segmentSpool) open() (*rawMember, error) { + i := len(s.members) + raw, err := os.Create(filepath.Join(s.dir, fmt.Sprintf("raw-%d", i))) + if err != nil { + return nil, fmt.Errorf("failed to create snapshot member: %w", err) + } + memberPath := filepath.Join(s.dir, fmt.Sprintf("member-%d", i)) + s.members = append(s.members, memberPath) + m := &rawMember{raw: raw, rawPath: raw.Name(), memberPath: memberPath} + m.tar = tar.NewWriter(m) + return m, nil +} + +// close finishes a raw member and queues its compression. Only the last member gets the +// end-of-archive marker. +func (s *segmentSpool) close(ctx context.Context, m *rawMember, last bool) error { + if last { + if err := m.tar.Close(); err != nil { + return fmt.Errorf("failed to finish snapshot tar: %w", err) + } + } else if err := m.tar.Flush(); err != nil { + return fmt.Errorf("failed to finish snapshot member: %w", err) + } + if err := m.raw.Close(); err != nil { + return fmt.Errorf("failed to finish snapshot member: %w", err) + } + s.g.Go(func() error { + s.slots <- struct{}{} + defer func() { <-s.slots }() + return compressRaw(ctx, m.rawPath, m.memberPath) + }) + return nil +} + +// compressRaw gzips the raw tar at rawPath into the member file at memberPath. +func compressRaw(ctx context.Context, rawPath, memberPath string) error { + raw, err := os.Open(rawPath) + if err != nil { + return err + } + defer raw.Close() + return spoolFile(ctx, memberPath, func(ctx context.Context, w io.Writer) error { + gz, err := gzip.NewWriterLevel(ctxWriter{ctx, w}, gzip.DefaultCompression) + if err != nil { + return err + } + if _, err := io.Copy(gz, raw); err != nil { + return fmt.Errorf("failed to create plain tarball: %w", err) + } + return gz.Close() + }) +} + // appendFile copies the file at path onto the end of w and returns the bytes copied. func appendFile(w io.Writer, path string) (int64, error) { f, err := os.Open(path) diff --git a/cmd/air/snapshot_upload.go b/cmd/air/snapshot_upload.go index b22362457ab..fd6903e2c75 100644 --- a/cmd/air/snapshot_upload.go +++ b/cmd/air/snapshot_upload.go @@ -139,19 +139,14 @@ func snapshotTarName(ctx context.Context, repoPath string, plan snapshotPlan) (s return fmt.Sprintf("%s_%s.tar.gz", dirName, computePlainTarKey(files)[:16]), files, nil } -// packageSnapshot writes the snapshot to tarball per the resolved plan: `git archive` -// of the pinned commit for git_archive, else a segmented plain tar of the pre-listed -// working-tree files. It returns the segment index for plain_tar and nil for -// git_archive, which has no sidecar yet. -func packageSnapshot(ctx context.Context, repoPath string, plan snapshotPlan, files []snapshotFile, tarball string) (*segmentIndex, error) { +// packageSnapshot writes the snapshot to tarball per the resolved plan, as a segmented +// tarball: the pinned commit's `git archive` for git_archive, else the pre-listed +// working-tree files. It returns the index that is uploaded alongside the tarball. +func packageSnapshot(ctx context.Context, repoPath string, plan snapshotPlan, files []snapshotFile, tarball string) (segmentIndex, error) { if plan.mode == modeGitArchive { - return nil, createGitArchiveSnapshot(ctx, newGitRepo(repoPath), plan.commitSHA, tarball, filepath.Base(repoPath), plan.includePaths, plan.subtreePrefix) + return createGitArchiveSnapshot(ctx, newGitRepo(repoPath), plan.commitSHA, tarball, filepath.Base(repoPath), plan.includePaths, plan.subtreePrefix) } - index, err := createPlainTarball(ctx, repoPath, tarball, files) - if err != nil { - return nil, err - } - return &index, nil + return createPlainTarball(ctx, repoPath, tarball, files) } // airArtifactInternalDir mirrors bundle/libraries.InternalDirName: CLI-uploaded @@ -180,7 +175,7 @@ func uploadSnapshotTarball(ctx context.Context, w *databricks.WorkspaceClient, r // upload the bytes now or reuse an object already in the store. remote := path.Join(uploadPath, tarName) - cached, err := snapshotCached(ctx, f, tarName, plan.mode == modePlainTar) + cached, err := snapshotCached(ctx, f, tarName) if err != nil { return result, err } @@ -223,9 +218,7 @@ func uploadSnapshotTarball(ctx context.Context, w *databricks.WorkspaceClient, r // The sidecar and tarball are written concurrently. The job is only submitted once both // succeed, and a later cache hit requires both (see snapshotCached). g, gctx := errgroup.WithContext(ctx) - if index != nil { - g.Go(func() error { return uploadSegmentIndex(gctx, f, tarName, index) }) - } + g.Go(func() error { return uploadSegmentIndex(gctx, f, tarName, &index) }) g.Go(func() error { if err := f.Write(gctx, tarName, file, filer.OverwriteIfExists, filer.CreateParentDirectories); err != nil { return fmt.Errorf("failed to upload snapshot %s: %w", tarName, err) @@ -241,14 +234,11 @@ func uploadSnapshotTarball(ctx context.Context, w *databricks.WorkspaceClient, r return result, nil } -// snapshotCached reports whether the snapshot is already in the store. For an indexed -// (plain_tar) snapshot both the tarball and its index sidecar must exist: a half-written -// pair from an interrupted upload is a miss, and the pair is re-uploaded. -func snapshotCached(ctx context.Context, store filer.Filer, tarName string, indexed bool) (bool, error) { - names := []string{tarName} - if indexed { - names = append(names, tarName+segmentIndexSuffix) - } +// snapshotCached reports whether the snapshot is already in the store. Both the tarball and +// its index sidecar must exist: a half-written pair from an interrupted upload is a miss, and +// the pair is re-uploaded. A tarball from before sidecars were written is therefore also a miss. +func snapshotCached(ctx context.Context, store filer.Filer, tarName string) (bool, error) { + names := []string{tarName, tarName + segmentIndexSuffix} exists := make([]bool, len(names)) g, gctx := errgroup.WithContext(ctx) for i, name := range names { diff --git a/cmd/air/testdata/cache_keys.json b/cmd/air/testdata/cache_keys.json index 06673499109..23ceba62c02 100644 --- a/cmd/air/testdata/cache_keys.json +++ b/cmd/air/testdata/cache_keys.json @@ -3,13 +3,13 @@ "name": "commit_no_paths", "commit_sha": "a3492b801c0ffee00000000000000000000dead", "include_paths": null, - "cache_key": "8d7bc445ac83dfe432a353718ae8b1ae40eb00c860352662e6df96b7fdf862a5" + "cache_key": "b372b0c84900bb4dc5a3dbe42f0ed0fc3a30a39096d0f1671ec7a732963880b3" }, { "name": "commit_empty_paths_none", "commit_sha": "0000000000000000000000000000000000000000", "include_paths": null, - "cache_key": "a07e0fa4d38f3c2d08c3ff6bbff0158cea01796aaaf5368b0d0e247cebd27c80" + "cache_key": "8c893a124d4c92ba09d6d69e459e3f1ee3704d18c8d3de49a6b82b2e8ac1332e" }, { "name": "single_path", @@ -17,7 +17,7 @@ "include_paths": [ "research" ], - "cache_key": "97bd562ad591af23b9a05651d88c01abbbcc24632e9a8f2534230a61492b9e0a" + "cache_key": "e1cb84d3e4cf51861a13e9f5693a5b842809f622120bf665529e52f6da58f96f" }, { "name": "multi_path_sorted", @@ -27,7 +27,7 @@ "b", "c" ], - "cache_key": "17d021577e3615e9a603a2f701f918d74ad16218efc4b2950ee20f6a18b1a174" + "cache_key": "6421973342e0e6a5fe15704d664372deb6b12c6d111e1d165c6916b49f0b44fc" }, { "name": "multi_path_unsorted", @@ -37,7 +37,7 @@ "a", "b" ], - "cache_key": "17d021577e3615e9a603a2f701f918d74ad16218efc4b2950ee20f6a18b1a174" + "cache_key": "6421973342e0e6a5fe15704d664372deb6b12c6d111e1d165c6916b49f0b44fc" }, { "name": "path_whitespace", @@ -46,7 +46,7 @@ " research ", " data " ], - "cache_key": "c6a6fa1d866ac90a5f371610ee08efa8aeb061a43b8a4acae30765db21f4155b" + "cache_key": "c6354f4b4f2c02e559926a079b6eb3fc560803cca426deb016398c56ee41f8ec" }, { "name": "nested_paths", @@ -56,13 +56,13 @@ "src/data", "README.md" ], - "cache_key": "2b18452b1019186a8e5b34af540a0545f96f24bc61771f58fa71fc218cb0275c" + "cache_key": "489892a2fdd3b869069b77f9e3b6dee5d51d85b649135b6654c3858305526182" }, { "name": "short_sha", "commit_sha": "a3492b8", "include_paths": null, - "cache_key": "16be4d7d83ecd01b8d3d274dafb07b3bccc72cf7f7299578e634cd01e1825345" + "cache_key": "5d0dc33af0fc697db6657113ecaf1b51cf8837a989c62604ba6c22852c8a900d" }, { "name": "dup_paths", @@ -72,6 +72,6 @@ "x", "y" ], - "cache_key": "407a5d8c3a0e9438d5abf653c4d3a43928c300b5630e2f83f435661b9fbd599e" + "cache_key": "20af9cc2df7443ce5e07e901337c9677185612fbedbd5e0e68272bbcef9c97bb" } ]