Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions cmd/air/runsubmit.go
Original file line number Diff line number Diff line change
Expand Up @@ -461,6 +461,9 @@ func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *run
return 0, "", err
}

// Opt-in snapshot GC runs once the run exists; it never changes the result.
gcAfterSubmit(ctx, w, cfg, base)

dashboardURL = strings.TrimRight(w.Config.Host, "/") + "/jobs/runs/" + strconv.FormatInt(runID, 10)
return runID, dashboardURL, nil
}
Expand Down
2 changes: 1 addition & 1 deletion cmd/air/snapshot_upload.go
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,7 @@ func uploadSnapshotTarball(ctx context.Context, w *databricks.WorkspaceClient, r
return result, err
}
if v2 {
err := uploadSnapshotV2(ctx, f, uploadPath, repoPath, files, &result)
err := uploadSnapshotV2(ctx, f, uploadPath, repoPath, files, &result, time.Now())
return result, err
}
}
Expand Down
85 changes: 71 additions & 14 deletions cmd/air/snapshot_v2.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
"runtime"
"slices"
"strings"
"sync/atomic"
"time"

"github.com/databricks/cli/libs/filer"
Expand Down Expand Up @@ -225,29 +226,31 @@ func v2ObjectName(id string) string {
return path.Join(v2ObjectsDir, id+".tar.gz")
}

// putV2Members makes every member an object in store and records its gzip size. An object
// that already exists is only sized: its id is its content, so it is never rewritten. At most
// v2UploadConcurrency members are compressed or uploaded at once.
func putV2Members(ctx context.Context, store filer.Filer, members []v2Member, tmp string) error {
// putV2Members makes every member an object in store and records its gzip size. At most
// v2UploadConcurrency members are compressed or uploaded at once. now is the time the snapshot
// is written, against which object age is judged.
func putV2Members(ctx context.Context, store filer.Filer, members []v2Member, tmp string, now time.Time) error {
g, gctx := errgroup.WithContext(ctx)
g.SetLimit(v2UploadConcurrency)
for i := range members {
g.Go(func() error {
return putV2Member(gctx, store, &members[i], tmp)
return putV2Member(gctx, store, &members[i], tmp, now)
})
}
return g.Wait()
}

// putV2Member writes one member as an object unless it already exists.
func putV2Member(ctx context.Context, store filer.Filer, m *v2Member, tmp string) error {
// putV2Member writes one member as an object unless a fresh copy already exists. An object
// younger than v2ReuseWindow is reused, since its id is its content. An older one is written
// again, which refreshes its mtime and keeps GC from deleting it under a reusing submit.
func putV2Member(ctx context.Context, store filer.Filer, m *v2Member, tmp string, now time.Time) error {
name := v2ObjectName(m.ID)
info, err := store.Stat(ctx, name)
switch {
case err == nil:
if err == nil && now.Sub(info.ModTime()) < v2ReuseWindow {
m.GzSize = info.Size()
return nil
case !errors.Is(err, fs.ErrNotExist):
}
if err != nil && !errors.Is(err, fs.ErrNotExist) {
return fmt.Errorf("failed to check snapshot object %s: %w", name, err)
}

Expand All @@ -270,6 +273,53 @@ func putV2Member(ctx context.Context, store filer.Filer, m *v2Member, tmp string
})
}

// renewV2Manifest renews an existing manifest whose objects are all present, by rewriting it
// unchanged. It reports false when the manifest is missing, unreadable, or references a missing
// object. The snapshot is then written in full, which also repairs a damaged manifest.
func renewV2Manifest(ctx context.Context, store filer.Filer, name string) (v2Manifest, bool, error) {
rc, err := store.Read(ctx, name)
if errors.Is(err, fs.ErrNotExist) {
return v2Manifest{}, false, nil
}
if err != nil {
return v2Manifest{}, false, fmt.Errorf("failed to read snapshot manifest %s: %w", name, err)
}
data, err := io.ReadAll(rc)
rc.Close()
if err != nil {
return v2Manifest{}, false, fmt.Errorf("failed to read snapshot manifest %s: %w", name, err)
}
var manifest v2Manifest
if err := json.Unmarshal(data, &manifest); err != nil {
log.Debugf(ctx, "snapshot manifest %s is unreadable, rewriting it: %v", name, err)
return v2Manifest{}, false, nil
}

var missing atomic.Bool
g, gctx := errgroup.WithContext(ctx)
g.SetLimit(v2UploadConcurrency)
for _, m := range manifest.Members {
g.Go(func() error {
_, err := store.Stat(gctx, v2ObjectName(m.ID))
if errors.Is(err, fs.ErrNotExist) {
missing.Store(true)
return nil
}
return err
})
}
if err := g.Wait(); err != nil {
return v2Manifest{}, false, fmt.Errorf("failed to check snapshot objects: %w", err)
}
if missing.Load() {
return v2Manifest{}, false, nil
}
if err := store.Write(ctx, name, bytes.NewReader(data), filer.OverwriteIfExists, filer.CreateParentDirectories); err != nil {
return v2Manifest{}, false, fmt.Errorf("failed to renew snapshot manifest %s: %w", name, err)
}
return manifest, true, nil
}

// withV2Retry runs op up to v2UploadAttempts times with exponential backoff. Context
// cancellation is never retried.
func withV2Retry(ctx context.Context, op func() error) error {
Expand Down Expand Up @@ -334,20 +384,27 @@ func previousV2K(ctx context.Context, store filer.Filer, component string) (int,
// uploadSnapshotV2 writes the working-tree snapshot in v2 form and sets result from it. The
// manifest is written last, after every object succeeded, so a visible manifest always refers
// to complete objects. An unchanged snapshot is found by its manifest name alone.
func uploadSnapshotV2(ctx context.Context, store filer.Filer, uploadPath, repoPath string, files []snapshotFile, result *snapshotResult) error {
func uploadSnapshotV2(ctx context.Context, store filer.Filer, uploadPath, repoPath string, files []snapshotFile, result *snapshotResult, now time.Time) error {
component := filepath.Base(repoPath)
manifestName := component + "_" + computePlainTarKey(files)[:16] + v2ManifestSuffix
remote := path.Join(uploadPath, manifestName)

cached, err := snapshotExists(ctx, store, manifestName)
// An unchanged snapshot renews its manifest's lease when all its objects are present. Renewal
// is a rewrite, which is what keeps GC from treating the manifest as abandoned.
previous, renewed, err := renewV2Manifest(ctx, store, manifestName)
if err != nil {
return err
}
if cached {
if renewed {
log.Debugf(ctx, "snapshot upload skipped; reusing %s", remote)
var gzTotal int64
for _, m := range previous.Members {
gzTotal += m.GzSize
}
result.CodeSourcePath = remote
result.PackagingDurationMs = new(int64(0))
result.UploadDurationMs = new(int64(0))
result.SizeBytes = &gzTotal
return nil
}

Expand Down Expand Up @@ -380,7 +437,7 @@ func uploadSnapshotV2(ctx context.Context, store filer.Filer, uploadPath, repoPa
result.PackagingDurationMs = new(time.Since(packagingStart).Milliseconds())

uploadStart := time.Now()
if err := putV2Members(ctx, store, members, tmp); err != nil {
if err := putV2Members(ctx, store, members, tmp, now); err != nil {
return err
}
manifest := v2Manifest{Format: v2ManifestFormat, Version: v2ManifestVersion, Component: component, K: k, Members: members}
Expand Down
236 changes: 236 additions & 0 deletions cmd/air/snapshot_v2_gc.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,236 @@
package aircmd

// Garbage collection for v2 snapshot objects (design section 4). It is off unless
// DATABRICKS_AIR_SNAPSHOT_GC is set, and it only ever touches the v2 layout under the snapshot
// directory: manifests named *.air.json and objects/<32 hex>.tar.gz.
//
// Safety rests on two windows. An object is reused only while younger than v2ReuseWindow (R).
// It is deleted only when older than v2GCAge (A) and not referenced by a manifest younger than
// v2ManifestLease (L). Because A - R is a day, an object reused at time t cannot be deleted
// before t + (A - R). The submit that reused it writes its manifest within that day.

import (
"bytes"
"context"
"encoding/json"
"errors"
"io/fs"
"path"
"regexp"
"strings"
"time"

"github.com/databricks/cli/libs/env"
"github.com/databricks/cli/libs/filer"
"github.com/databricks/cli/libs/log"
"github.com/databricks/databricks-sdk-go"
)

const (
// snapshotGCEnv opts in to GC after a submit. It is unset by default.
snapshotGCEnv = "DATABRICKS_AIR_SNAPSHOT_GC"

v2ReuseWindow = 7 * 24 * time.Hour
v2GCAge = 8 * 24 * time.Hour
v2ManifestLease = 30 * 24 * time.Hour
v2GCInterval = 24 * time.Hour
v2GCTimeBox = 3 * time.Second
v2GCMarkerName = ".air-gc.marker"
)

// v2ObjectFile matches the object files GC may delete, relative to objects/.
var v2ObjectFile = regexp.MustCompile(`^[0-9a-f]{32}\.tar\.gz$`)

// v2GCPlan is what a GC run would delete. Names are relative to the snapshot directory.
type v2GCPlan struct {
Manifests []string
Objects []string
// ObjectsKept counts objects that are referenced or too young to delete.
ObjectsKept int
}

// planV2GC lists the v2 layout in store and decides what a GC run at now would delete. It
// deletes nothing. Any listing or manifest that cannot be read fails the plan, since an
// unread manifest could reference any object.
func planV2GC(ctx context.Context, store filer.Filer, now time.Time) (v2GCPlan, error) {
var plan v2GCPlan
entries, err := store.ReadDir(ctx, ".")
if err != nil && !errors.Is(err, fs.ErrNotExist) {
return plan, err
}
referenced := map[string]bool{}
for _, e := range entries {
if e.IsDir() || !strings.HasSuffix(e.Name(), v2ManifestSuffix) {
continue
}
info, err := e.Info()
if err != nil {
return v2GCPlan{}, err
}
if now.Sub(info.ModTime()) >= v2ManifestLease {
plan.Manifests = append(plan.Manifests, e.Name())
continue
}
ids, err := readV2ManifestIDs(ctx, store, e.Name())
if err != nil {
return v2GCPlan{}, err
}
for _, id := range ids {
referenced[id] = true
}
}

objects, err := store.ReadDir(ctx, v2ObjectsDir)
if err != nil && !errors.Is(err, fs.ErrNotExist) {
return v2GCPlan{}, err
}
for _, o := range objects {
if o.IsDir() || !v2ObjectFile.MatchString(o.Name()) {
continue
}
info, err := o.Info()
if err != nil {
return v2GCPlan{}, err
}
age := now.Sub(info.ModTime())
id := strings.TrimSuffix(o.Name(), ".tar.gz")
if referenced[id] || age < v2GCAge {
plan.ObjectsKept++
continue
}
plan.Objects = append(plan.Objects, path.Join(v2ObjectsDir, o.Name()))
}
return plan, nil
}

// readV2ManifestIDs returns the member ids of the manifest at name.
func readV2ManifestIDs(ctx context.Context, store filer.Filer, name string) ([]string, error) {
rc, err := store.Read(ctx, name)
if err != nil {
return nil, err
}
defer rc.Close()
var manifest v2Manifest
if err := json.NewDecoder(rc).Decode(&manifest); err != nil {
return nil, err
}
ids := make([]string, 0, len(manifest.Members))
for _, m := range manifest.Members {
ids = append(ids, m.ID)
}
return ids, nil
}

// applyV2GC deletes the planned names. Each object is checked again just before its delete, and
// it is kept if a submit has rewritten it since the plan. A failed delete is returned after the
// rest are attempted; partial progress is safe because each delete is independently justified.
func applyV2GC(ctx context.Context, store filer.Filer, plan v2GCPlan, now time.Time) error {
var firstErr error
for _, name := range plan.Manifests {
// Recheck the lease: a submit may have renewed the manifest since the plan.
info, err := store.Stat(ctx, name)
if errors.Is(err, fs.ErrNotExist) {
continue
}
if err != nil {
firstErr = errOr(firstErr, err)
continue
}
if now.Sub(info.ModTime()) < v2ManifestLease {
continue
}
if err := store.Delete(ctx, name); err != nil && !errors.Is(err, fs.ErrNotExist) {
firstErr = errOr(firstErr, err)
}
}
for _, name := range plan.Objects {
info, err := store.Stat(ctx, name)
if errors.Is(err, fs.ErrNotExist) {
continue
}
if err != nil {
firstErr = errOr(firstErr, err)
continue
}
if now.Sub(info.ModTime()) < v2GCAge {
continue
}
if err := store.Delete(ctx, name); err != nil && !errors.Is(err, fs.ErrNotExist) {
firstErr = errOr(firstErr, err)
}
}
return firstErr
}

// errOr returns first if set, else err.
func errOr(first, err error) error {
if first != nil {
return first
}
return err
}

// gcEnabled reports whether GC is opted in. Unset, or any value env.GetBool treats as false, is off.
func gcEnabled(ctx context.Context) bool {
on, _ := env.GetBool(ctx, snapshotGCEnv)
return on
}

// gcDue reports whether the last GC run is at least v2GCInterval old, by the marker's mtime.
func gcDue(ctx context.Context, store filer.Filer, now time.Time) (bool, error) {
info, err := store.Stat(ctx, v2GCMarkerName)
if errors.Is(err, fs.ErrNotExist) {
return true, nil
}
if err != nil {
return false, err
}
return now.Sub(info.ModTime()) >= v2GCInterval, nil
}

// runV2GC is the best-effort GC run after a submit: it runs at most once per v2GCInterval per
// store, and all its errors are logged at debug. It never fails the submit. The caller bounds
// ctx with v2GCTimeBox; if the box expires while planning, nothing is deleted.
func runV2GC(ctx context.Context, store filer.Filer, now time.Time) {
due, err := gcDue(ctx, store, now)
if err != nil {
log.Debugf(ctx, "snapshot GC skipped: %v", err)
return
}
if !due {
return
}
if err := store.Write(ctx, v2GCMarkerName, bytes.NewReader(nil), filer.OverwriteIfExists, filer.CreateParentDirectories); err != nil {
log.Debugf(ctx, "snapshot GC skipped: %v", err)
return
}
plan, err := planV2GC(ctx, store, now)
if err != nil {
log.Debugf(ctx, "snapshot GC planning failed; nothing deleted: %v", err)
return
}
if err := applyV2GC(ctx, store, plan, now); err != nil {
log.Debugf(ctx, "snapshot GC incomplete: %v", err)
}
log.Debugf(ctx, "snapshot GC deleted %d manifests and %d objects, kept %d objects", len(plan.Manifests), len(plan.Objects), plan.ObjectsKept)
}

// gcAfterSubmit runs GC for the snapshot directory of a submitted run, when GC is opted in. It
// runs after the run exists, and is bounded by v2GCTimeBox.
func gcAfterSubmit(ctx context.Context, w *databricks.WorkspaceClient, cfg *runConfig, base string) {
if !gcEnabled(ctx) || cfg.CodeSource == nil || cfg.CodeSource.Snapshot == nil {
return
}
artifactPath := path.Join(base, ".air", "repo_snapshots")
if cfg.CodeSource.Snapshot.RemoteVolume != nil {
artifactPath = *cfg.CodeSource.Snapshot.RemoteVolume
}
store, _, err := snapshotUploadFiler(ctx, w, artifactPath)
if err != nil {
log.Debugf(ctx, "snapshot GC skipped: %v", err)
return
}
gcCtx, cancel := context.WithTimeout(ctx, v2GCTimeBox)
defer cancel()
runV2GC(gcCtx, store, time.Now())
}
Loading
Loading