diff --git a/cmd/air/runsubmit.go b/cmd/air/runsubmit.go index 451772da098..88645e1fab1 100644 --- a/cmd/air/runsubmit.go +++ b/cmd/air/runsubmit.go @@ -461,6 +461,9 @@ func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *run return 0, "", err } + // Opt-in snapshot GC runs once the run exists; it never changes the result. + gcAfterSubmit(ctx, w, cfg, base) + dashboardURL = strings.TrimRight(w.Config.Host, "/") + "/jobs/runs/" + strconv.FormatInt(runID, 10) return runID, dashboardURL, nil } diff --git a/cmd/air/snapshot_upload.go b/cmd/air/snapshot_upload.go index c21b5b7c0c3..64bb5e7cb36 100644 --- a/cmd/air/snapshot_upload.go +++ b/cmd/air/snapshot_upload.go @@ -179,7 +179,7 @@ func uploadSnapshotTarball(ctx context.Context, w *databricks.WorkspaceClient, r return result, err } if v2 { - err := uploadSnapshotV2(ctx, f, uploadPath, repoPath, files, &result) + err := uploadSnapshotV2(ctx, f, uploadPath, repoPath, files, &result, time.Now()) return result, err } } diff --git a/cmd/air/snapshot_v2.go b/cmd/air/snapshot_v2.go index 1026f7091ea..17cb01cbd86 100644 --- a/cmd/air/snapshot_v2.go +++ b/cmd/air/snapshot_v2.go @@ -23,6 +23,7 @@ import ( "runtime" "slices" "strings" + "sync/atomic" "time" "github.com/databricks/cli/libs/filer" @@ -225,29 +226,31 @@ func v2ObjectName(id string) string { return path.Join(v2ObjectsDir, id+".tar.gz") } -// putV2Members makes every member an object in store and records its gzip size. An object -// that already exists is only sized: its id is its content, so it is never rewritten. At most -// v2UploadConcurrency members are compressed or uploaded at once. -func putV2Members(ctx context.Context, store filer.Filer, members []v2Member, tmp string) error { +// putV2Members makes every member an object in store and records its gzip size. At most +// v2UploadConcurrency members are compressed or uploaded at once. now is the time the snapshot +// is written, against which object age is judged. +func putV2Members(ctx context.Context, store filer.Filer, members []v2Member, tmp string, now time.Time) error { g, gctx := errgroup.WithContext(ctx) g.SetLimit(v2UploadConcurrency) for i := range members { g.Go(func() error { - return putV2Member(gctx, store, &members[i], tmp) + return putV2Member(gctx, store, &members[i], tmp, now) }) } return g.Wait() } -// putV2Member writes one member as an object unless it already exists. -func putV2Member(ctx context.Context, store filer.Filer, m *v2Member, tmp string) error { +// putV2Member writes one member as an object unless a fresh copy already exists. An object +// younger than v2ReuseWindow is reused, since its id is its content. An older one is written +// again, which refreshes its mtime and keeps GC from deleting it under a reusing submit. +func putV2Member(ctx context.Context, store filer.Filer, m *v2Member, tmp string, now time.Time) error { name := v2ObjectName(m.ID) info, err := store.Stat(ctx, name) - switch { - case err == nil: + if err == nil && now.Sub(info.ModTime()) < v2ReuseWindow { m.GzSize = info.Size() return nil - case !errors.Is(err, fs.ErrNotExist): + } + if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("failed to check snapshot object %s: %w", name, err) } @@ -270,6 +273,53 @@ func putV2Member(ctx context.Context, store filer.Filer, m *v2Member, tmp string }) } +// renewV2Manifest renews an existing manifest whose objects are all present, by rewriting it +// unchanged. It reports false when the manifest is missing, unreadable, or references a missing +// object. The snapshot is then written in full, which also repairs a damaged manifest. +func renewV2Manifest(ctx context.Context, store filer.Filer, name string) (v2Manifest, bool, error) { + rc, err := store.Read(ctx, name) + if errors.Is(err, fs.ErrNotExist) { + return v2Manifest{}, false, nil + } + if err != nil { + return v2Manifest{}, false, fmt.Errorf("failed to read snapshot manifest %s: %w", name, err) + } + data, err := io.ReadAll(rc) + rc.Close() + if err != nil { + return v2Manifest{}, false, fmt.Errorf("failed to read snapshot manifest %s: %w", name, err) + } + var manifest v2Manifest + if err := json.Unmarshal(data, &manifest); err != nil { + log.Debugf(ctx, "snapshot manifest %s is unreadable, rewriting it: %v", name, err) + return v2Manifest{}, false, nil + } + + var missing atomic.Bool + g, gctx := errgroup.WithContext(ctx) + g.SetLimit(v2UploadConcurrency) + for _, m := range manifest.Members { + g.Go(func() error { + _, err := store.Stat(gctx, v2ObjectName(m.ID)) + if errors.Is(err, fs.ErrNotExist) { + missing.Store(true) + return nil + } + return err + }) + } + if err := g.Wait(); err != nil { + return v2Manifest{}, false, fmt.Errorf("failed to check snapshot objects: %w", err) + } + if missing.Load() { + return v2Manifest{}, false, nil + } + if err := store.Write(ctx, name, bytes.NewReader(data), filer.OverwriteIfExists, filer.CreateParentDirectories); err != nil { + return v2Manifest{}, false, fmt.Errorf("failed to renew snapshot manifest %s: %w", name, err) + } + return manifest, true, nil +} + // withV2Retry runs op up to v2UploadAttempts times with exponential backoff. Context // cancellation is never retried. func withV2Retry(ctx context.Context, op func() error) error { @@ -334,20 +384,27 @@ func previousV2K(ctx context.Context, store filer.Filer, component string) (int, // uploadSnapshotV2 writes the working-tree snapshot in v2 form and sets result from it. The // manifest is written last, after every object succeeded, so a visible manifest always refers // to complete objects. An unchanged snapshot is found by its manifest name alone. -func uploadSnapshotV2(ctx context.Context, store filer.Filer, uploadPath, repoPath string, files []snapshotFile, result *snapshotResult) error { +func uploadSnapshotV2(ctx context.Context, store filer.Filer, uploadPath, repoPath string, files []snapshotFile, result *snapshotResult, now time.Time) error { component := filepath.Base(repoPath) manifestName := component + "_" + computePlainTarKey(files)[:16] + v2ManifestSuffix remote := path.Join(uploadPath, manifestName) - cached, err := snapshotExists(ctx, store, manifestName) + // An unchanged snapshot renews its manifest's lease when all its objects are present. Renewal + // is a rewrite, which is what keeps GC from treating the manifest as abandoned. + previous, renewed, err := renewV2Manifest(ctx, store, manifestName) if err != nil { return err } - if cached { + if renewed { log.Debugf(ctx, "snapshot upload skipped; reusing %s", remote) + var gzTotal int64 + for _, m := range previous.Members { + gzTotal += m.GzSize + } result.CodeSourcePath = remote result.PackagingDurationMs = new(int64(0)) result.UploadDurationMs = new(int64(0)) + result.SizeBytes = &gzTotal return nil } @@ -380,7 +437,7 @@ func uploadSnapshotV2(ctx context.Context, store filer.Filer, uploadPath, repoPa result.PackagingDurationMs = new(time.Since(packagingStart).Milliseconds()) uploadStart := time.Now() - if err := putV2Members(ctx, store, members, tmp); err != nil { + if err := putV2Members(ctx, store, members, tmp, now); err != nil { return err } manifest := v2Manifest{Format: v2ManifestFormat, Version: v2ManifestVersion, Component: component, K: k, Members: members} diff --git a/cmd/air/snapshot_v2_gc.go b/cmd/air/snapshot_v2_gc.go new file mode 100644 index 00000000000..af9c25408f2 --- /dev/null +++ b/cmd/air/snapshot_v2_gc.go @@ -0,0 +1,236 @@ +package aircmd + +// Garbage collection for v2 snapshot objects (design section 4). It is off unless +// DATABRICKS_AIR_SNAPSHOT_GC is set, and it only ever touches the v2 layout under the snapshot +// directory: manifests named *.air.json and objects/<32 hex>.tar.gz. +// +// Safety rests on two windows. An object is reused only while younger than v2ReuseWindow (R). +// It is deleted only when older than v2GCAge (A) and not referenced by a manifest younger than +// v2ManifestLease (L). Because A - R is a day, an object reused at time t cannot be deleted +// before t + (A - R). The submit that reused it writes its manifest within that day. + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io/fs" + "path" + "regexp" + "strings" + "time" + + "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/filer" + "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go" +) + +const ( + // snapshotGCEnv opts in to GC after a submit. It is unset by default. + snapshotGCEnv = "DATABRICKS_AIR_SNAPSHOT_GC" + + v2ReuseWindow = 7 * 24 * time.Hour + v2GCAge = 8 * 24 * time.Hour + v2ManifestLease = 30 * 24 * time.Hour + v2GCInterval = 24 * time.Hour + v2GCTimeBox = 3 * time.Second + v2GCMarkerName = ".air-gc.marker" +) + +// v2ObjectFile matches the object files GC may delete, relative to objects/. +var v2ObjectFile = regexp.MustCompile(`^[0-9a-f]{32}\.tar\.gz$`) + +// v2GCPlan is what a GC run would delete. Names are relative to the snapshot directory. +type v2GCPlan struct { + Manifests []string + Objects []string + // ObjectsKept counts objects that are referenced or too young to delete. + ObjectsKept int +} + +// planV2GC lists the v2 layout in store and decides what a GC run at now would delete. It +// deletes nothing. Any listing or manifest that cannot be read fails the plan, since an +// unread manifest could reference any object. +func planV2GC(ctx context.Context, store filer.Filer, now time.Time) (v2GCPlan, error) { + var plan v2GCPlan + entries, err := store.ReadDir(ctx, ".") + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return plan, err + } + referenced := map[string]bool{} + for _, e := range entries { + if e.IsDir() || !strings.HasSuffix(e.Name(), v2ManifestSuffix) { + continue + } + info, err := e.Info() + if err != nil { + return v2GCPlan{}, err + } + if now.Sub(info.ModTime()) >= v2ManifestLease { + plan.Manifests = append(plan.Manifests, e.Name()) + continue + } + ids, err := readV2ManifestIDs(ctx, store, e.Name()) + if err != nil { + return v2GCPlan{}, err + } + for _, id := range ids { + referenced[id] = true + } + } + + objects, err := store.ReadDir(ctx, v2ObjectsDir) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return v2GCPlan{}, err + } + for _, o := range objects { + if o.IsDir() || !v2ObjectFile.MatchString(o.Name()) { + continue + } + info, err := o.Info() + if err != nil { + return v2GCPlan{}, err + } + age := now.Sub(info.ModTime()) + id := strings.TrimSuffix(o.Name(), ".tar.gz") + if referenced[id] || age < v2GCAge { + plan.ObjectsKept++ + continue + } + plan.Objects = append(plan.Objects, path.Join(v2ObjectsDir, o.Name())) + } + return plan, nil +} + +// readV2ManifestIDs returns the member ids of the manifest at name. +func readV2ManifestIDs(ctx context.Context, store filer.Filer, name string) ([]string, error) { + rc, err := store.Read(ctx, name) + if err != nil { + return nil, err + } + defer rc.Close() + var manifest v2Manifest + if err := json.NewDecoder(rc).Decode(&manifest); err != nil { + return nil, err + } + ids := make([]string, 0, len(manifest.Members)) + for _, m := range manifest.Members { + ids = append(ids, m.ID) + } + return ids, nil +} + +// applyV2GC deletes the planned names. Each object is checked again just before its delete, and +// it is kept if a submit has rewritten it since the plan. A failed delete is returned after the +// rest are attempted; partial progress is safe because each delete is independently justified. +func applyV2GC(ctx context.Context, store filer.Filer, plan v2GCPlan, now time.Time) error { + var firstErr error + for _, name := range plan.Manifests { + // Recheck the lease: a submit may have renewed the manifest since the plan. + info, err := store.Stat(ctx, name) + if errors.Is(err, fs.ErrNotExist) { + continue + } + if err != nil { + firstErr = errOr(firstErr, err) + continue + } + if now.Sub(info.ModTime()) < v2ManifestLease { + continue + } + if err := store.Delete(ctx, name); err != nil && !errors.Is(err, fs.ErrNotExist) { + firstErr = errOr(firstErr, err) + } + } + for _, name := range plan.Objects { + info, err := store.Stat(ctx, name) + if errors.Is(err, fs.ErrNotExist) { + continue + } + if err != nil { + firstErr = errOr(firstErr, err) + continue + } + if now.Sub(info.ModTime()) < v2GCAge { + continue + } + if err := store.Delete(ctx, name); err != nil && !errors.Is(err, fs.ErrNotExist) { + firstErr = errOr(firstErr, err) + } + } + return firstErr +} + +// errOr returns first if set, else err. +func errOr(first, err error) error { + if first != nil { + return first + } + return err +} + +// gcEnabled reports whether GC is opted in. Unset, or any value env.GetBool treats as false, is off. +func gcEnabled(ctx context.Context) bool { + on, _ := env.GetBool(ctx, snapshotGCEnv) + return on +} + +// gcDue reports whether the last GC run is at least v2GCInterval old, by the marker's mtime. +func gcDue(ctx context.Context, store filer.Filer, now time.Time) (bool, error) { + info, err := store.Stat(ctx, v2GCMarkerName) + if errors.Is(err, fs.ErrNotExist) { + return true, nil + } + if err != nil { + return false, err + } + return now.Sub(info.ModTime()) >= v2GCInterval, nil +} + +// runV2GC is the best-effort GC run after a submit: it runs at most once per v2GCInterval per +// store, and all its errors are logged at debug. It never fails the submit. The caller bounds +// ctx with v2GCTimeBox; if the box expires while planning, nothing is deleted. +func runV2GC(ctx context.Context, store filer.Filer, now time.Time) { + due, err := gcDue(ctx, store, now) + if err != nil { + log.Debugf(ctx, "snapshot GC skipped: %v", err) + return + } + if !due { + return + } + if err := store.Write(ctx, v2GCMarkerName, bytes.NewReader(nil), filer.OverwriteIfExists, filer.CreateParentDirectories); err != nil { + log.Debugf(ctx, "snapshot GC skipped: %v", err) + return + } + plan, err := planV2GC(ctx, store, now) + if err != nil { + log.Debugf(ctx, "snapshot GC planning failed; nothing deleted: %v", err) + return + } + if err := applyV2GC(ctx, store, plan, now); err != nil { + log.Debugf(ctx, "snapshot GC incomplete: %v", err) + } + log.Debugf(ctx, "snapshot GC deleted %d manifests and %d objects, kept %d objects", len(plan.Manifests), len(plan.Objects), plan.ObjectsKept) +} + +// gcAfterSubmit runs GC for the snapshot directory of a submitted run, when GC is opted in. It +// runs after the run exists, and is bounded by v2GCTimeBox. +func gcAfterSubmit(ctx context.Context, w *databricks.WorkspaceClient, cfg *runConfig, base string) { + if !gcEnabled(ctx) || cfg.CodeSource == nil || cfg.CodeSource.Snapshot == nil { + return + } + artifactPath := path.Join(base, ".air", "repo_snapshots") + if cfg.CodeSource.Snapshot.RemoteVolume != nil { + artifactPath = *cfg.CodeSource.Snapshot.RemoteVolume + } + store, _, err := snapshotUploadFiler(ctx, w, artifactPath) + if err != nil { + log.Debugf(ctx, "snapshot GC skipped: %v", err) + return + } + gcCtx, cancel := context.WithTimeout(ctx, v2GCTimeBox) + defer cancel() + runV2GC(gcCtx, store, time.Now()) +} diff --git a/cmd/air/snapshot_v2_gc_test.go b/cmd/air/snapshot_v2_gc_test.go new file mode 100644 index 00000000000..e5851b0d7d2 --- /dev/null +++ b/cmd/air/snapshot_v2_gc_test.go @@ -0,0 +1,464 @@ +package aircmd + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "io/fs" + "path" + "slices" + "strings" + "sync" + "testing" + "time" + + "github.com/databricks/cli/libs/filer" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fakeClock is the time source for the GC and writer tests. +type fakeClock struct{ now time.Time } + +func (c *fakeClock) Now() time.Time { return c.now } + +func (c *fakeClock) Advance(d time.Duration) { c.now = c.now.Add(d) } + +// memFile is one file in memFiler, stamped with the clock time of its last write. +type memFile struct { + data []byte + mtime time.Time +} + +// memFiler is an in-memory filer.Filer. Files are keyed by slash path relative to the store +// root, and ReadDir lists the direct children of a directory. +type memFiler struct { + mu sync.Mutex + clock *fakeClock + files map[string]memFile + // writes counts writes by object name, for the writer tests. + writes map[string]int + // blockDir, when set, makes ReadDir of that directory wait for the context. + blockDir string +} + +func newMemFiler(clock *fakeClock) *memFiler { + return &memFiler{clock: clock, files: map[string]memFile{}, writes: map[string]int{}} +} + +func memKey(p string) string { + return path.Clean(strings.TrimPrefix(p, "/")) +} + +func (f *memFiler) Write(ctx context.Context, p string, r io.Reader, _ ...filer.WriteMode) error { + data, err := io.ReadAll(r) + if err != nil { + return err + } + f.mu.Lock() + defer f.mu.Unlock() + f.files[memKey(p)] = memFile{data: data, mtime: f.clock.Now()} + f.writes[path.Base(p)]++ + return nil +} + +func (f *memFiler) Read(_ context.Context, p string) (io.ReadCloser, error) { + f.mu.Lock() + defer f.mu.Unlock() + file, ok := f.files[memKey(p)] + if !ok { + return nil, &fs.PathError{Op: "read", Path: p, Err: fs.ErrNotExist} + } + return io.NopCloser(bytes.NewReader(file.data)), nil +} + +func (f *memFiler) Delete(_ context.Context, p string, _ ...filer.DeleteMode) error { + f.mu.Lock() + defer f.mu.Unlock() + if _, ok := f.files[memKey(p)]; !ok { + return &fs.PathError{Op: "delete", Path: p, Err: fs.ErrNotExist} + } + delete(f.files, memKey(p)) + return nil +} + +func (f *memFiler) Stat(_ context.Context, p string) (fs.FileInfo, error) { + f.mu.Lock() + defer f.mu.Unlock() + file, ok := f.files[memKey(p)] + if !ok { + return nil, &fs.PathError{Op: "stat", Path: p, Err: fs.ErrNotExist} + } + return memInfo{name: path.Base(memKey(p)), size: int64(len(file.data)), mtime: file.mtime}, nil +} + +func (f *memFiler) ReadDir(ctx context.Context, dir string) ([]fs.DirEntry, error) { + if dir == f.blockDir { + <-ctx.Done() + return nil, ctx.Err() + } + f.mu.Lock() + defer f.mu.Unlock() + parent := memKey(dir) + var entries []fs.DirEntry + for k, file := range f.files { + if path.Dir(k) != parent { + continue + } + entries = append(entries, memEntry{info: memInfo{name: path.Base(k), size: int64(len(file.data)), mtime: file.mtime}}) + } + slices.SortFunc(entries, func(a, b fs.DirEntry) int { return strings.Compare(a.Name(), b.Name()) }) + return entries, nil +} + +func (f *memFiler) Mkdir(context.Context, string) error { return nil } + +// names returns the keys currently in the store, sorted. +func (f *memFiler) names() []string { + f.mu.Lock() + defer f.mu.Unlock() + var out []string + for k := range f.files { + out = append(out, k) + } + slices.Sort(out) + return out +} + +type memInfo struct { + name string + size int64 + mtime time.Time +} + +func (i memInfo) Name() string { return i.name } +func (i memInfo) Size() int64 { return i.size } +func (i memInfo) Mode() fs.FileMode { return 0o644 } +func (i memInfo) ModTime() time.Time { return i.mtime } +func (i memInfo) IsDir() bool { return false } +func (i memInfo) Sys() any { return nil } + +type memEntry struct{ info memInfo } + +func (e memEntry) Name() string { return e.info.name } +func (e memEntry) IsDir() bool { return false } +func (e memEntry) Type() fs.FileMode { return 0 } +func (e memEntry) Info() (fs.FileInfo, error) { return e.info, nil } + +// v2TestID is a 32-hex object id for the fixtures. +func v2TestID(n int) string { + return fmt.Sprintf("%032x", n) +} + +// seedObject writes an object as of the clock's current time. +func seedObject(t *testing.T, f *memFiler, id string) { + t.Helper() + require.NoError(t, f.Write(t.Context(), v2ObjectName(id), bytes.NewReader([]byte("gz")), filer.OverwriteIfExists, filer.CreateParentDirectories)) +} + +// seedManifest writes a v2 manifest that references ids, as of the clock's current time. +func seedManifest(t *testing.T, f *memFiler, name string, ids ...string) { + t.Helper() + m := v2Manifest{Format: v2ManifestFormat, Version: v2ManifestVersion, Component: "code", K: 1} + for _, id := range ids { + m.Members = append(m.Members, v2Member{ID: id, GzSize: 2, TarSize: 1024}) + } + data, err := json.Marshal(m) + require.NoError(t, err) + require.NoError(t, f.Write(t.Context(), name, bytes.NewReader(data), filer.OverwriteIfExists, filer.CreateParentDirectories)) +} + +func newGCClock() *fakeClock { + return &fakeClock{now: time.Date(2026, 11, 1, 12, 0, 0, 0, time.UTC)} +} + +// Constants must keep the reuse window below GC age by the margin the safety argument uses. +func TestV2GCWindowsKeepTheSafetyMargin(t *testing.T) { + assert.Equal(t, 7*24*time.Hour, v2ReuseWindow) + assert.Equal(t, 8*24*time.Hour, v2GCAge) + assert.Equal(t, 30*24*time.Hour, v2ManifestLease) + assert.Equal(t, 24*time.Hour, v2GCInterval) + assert.Equal(t, 24*time.Hour, v2GCAge-v2ReuseWindow) +} + +// An object reused at time T has an mtime above T - R, so it cannot be deleted before +// T + (A - R). A GC run that lists the store in that window keeps it, even before the reusing +// submit writes its manifest. +func TestV2GCRaceReusedObjectNotDeletableBeforeMargin(t *testing.T) { + clock := newGCClock() + f := newMemFiler(clock) + id := v2TestID(1) + + // The object was written just inside the reuse window when the submit at T reuses it. + reuse := clock.Now() + clock.now = reuse.Add(-v2ReuseWindow + time.Second) + seedObject(t, f, id) + clock.now = reuse + + deadline := reuse.Add(v2GCAge - v2ReuseWindow) + for _, at := range []time.Duration{0, time.Hour, 12 * time.Hour, deadline.Sub(reuse) - time.Minute} { + plan, err := planV2GC(t.Context(), f, reuse.Add(at)) + require.NoError(t, err) + assert.NotContains(t, plan.Objects, v2ObjectName(id), "deleted too early at +%v", at) + } + // Past the margin the same object is deletable, which bounds the window from above. + plan, err := planV2GC(t.Context(), f, deadline.Add(2*time.Second)) + require.NoError(t, err) + assert.Contains(t, plan.Objects, v2ObjectName(id)) +} + +// A manifest younger than L keeps the objects it references, however old they are. An +// unreferenced object older than A is deleted. +func TestV2GCLeasedManifestKeepsItsObjects(t *testing.T) { + clock := newGCClock() + f := newMemFiler(clock) + kept, orphan := v2TestID(1), v2TestID(2) + + seedObject(t, f, kept) + seedObject(t, f, orphan) + clock.Advance(20 * 24 * time.Hour) + seedManifest(t, f, "code_aaaaaaaaaaaaaaaa.air.json", kept) + clock.Advance(v2GCAge) // objects are now 28 days old, past A + + plan, err := planV2GC(t.Context(), f, clock.Now()) + require.NoError(t, err) + assert.Empty(t, plan.Manifests) + assert.Equal(t, []string{v2ObjectName(orphan)}, plan.Objects) + assert.Equal(t, 1, plan.ObjectsKept) + + require.NoError(t, applyV2GC(t.Context(), f, plan, clock.Now())) + assert.Contains(t, f.names(), v2ObjectName(kept)) + assert.NotContains(t, f.names(), v2ObjectName(orphan)) +} + +// A manifest older than L is deleted, and its objects go with it once older than A. +func TestV2GCAbandonedManifestAndItsObjectsAreDeleted(t *testing.T) { + clock := newGCClock() + f := newMemFiler(clock) + id := v2TestID(3) + seedObject(t, f, id) + seedManifest(t, f, "code_bbbbbbbbbbbbbbbb.air.json", id) + clock.Advance(v2ManifestLease + time.Hour) + + plan, err := planV2GC(t.Context(), f, clock.Now()) + require.NoError(t, err) + assert.Equal(t, []string{"code_bbbbbbbbbbbbbbbb.air.json"}, plan.Manifests) + assert.Equal(t, []string{v2ObjectName(id)}, plan.Objects) + + require.NoError(t, applyV2GC(t.Context(), f, plan, clock.Now())) + assert.Empty(t, f.names()) +} + +// An object older than A with no manifest is an orphan from an interrupted submit: deleted. A +// younger one is kept. +func TestV2GCOrphansDeletedAfterGCAge(t *testing.T) { + clock := newGCClock() + f := newMemFiler(clock) + old, young := v2TestID(4), v2TestID(5) + seedObject(t, f, old) + clock.Advance(v2GCAge + time.Hour) + seedObject(t, f, young) + clock.Advance(v2GCAge - 2*time.Hour) + + plan, err := planV2GC(t.Context(), f, clock.Now()) + require.NoError(t, err) + assert.Equal(t, []string{v2ObjectName(old)}, plan.Objects) + assert.Equal(t, 1, plan.ObjectsKept) +} + +// Nothing outside the v2 layout is touched: v1 tarballs and sidecars, notes, and objects with +// names that are not 32 hex characters. Even ancient ones stay. +func TestV2GCLeavesOtherFilesAlone(t *testing.T) { + clock := newGCClock() + f := newMemFiler(clock) + others := []string{ + "code_0123456789abcdef.tar.gz", + "code_0123456789abcdef.tar.gz.idx.json", + "notes.txt", + "code_0123456789abcdef.air.json.bak", + "objects/notahexnameatall.tar.gz", + "objects/" + v2TestID(6) + ".gz", + "objects/sub/" + v2TestID(7) + ".tar.gz", + } + for _, name := range others { + require.NoError(t, f.Write(t.Context(), name, bytes.NewReader([]byte("x")), filer.CreateParentDirectories)) + } + seedObject(t, f, v2TestID(8)) + clock.Advance(365 * 24 * time.Hour) + + plan, err := planV2GC(t.Context(), f, clock.Now()) + require.NoError(t, err) + require.NoError(t, applyV2GC(t.Context(), f, plan, clock.Now())) + + assert.Equal(t, []string{v2ObjectName(v2TestID(8))}, plan.Objects) + for _, name := range others { + assert.Contains(t, f.names(), name) + } +} + +// A manifest renewed between the plan and the apply is kept, and so is an object rewritten by a +// submit in that window. +func TestV2GCApplyRechecksAgeBeforeDelete(t *testing.T) { + clock := newGCClock() + f := newMemFiler(clock) + id := v2TestID(9) + seedObject(t, f, id) + seedManifest(t, f, "code_cccccccccccccccc.air.json") + clock.Advance(v2ManifestLease + time.Hour) + + plan, err := planV2GC(t.Context(), f, clock.Now()) + require.NoError(t, err) + require.Contains(t, plan.Objects, v2ObjectName(id)) + require.Contains(t, plan.Manifests, "code_cccccccccccccccc.air.json") + + // A submit rewrites the object and the manifest before the apply runs. + seedObject(t, f, id) + seedManifest(t, f, "code_cccccccccccccccc.air.json") + require.NoError(t, applyV2GC(t.Context(), f, plan, clock.Now())) + assert.Contains(t, f.names(), v2ObjectName(id)) + assert.Contains(t, f.names(), "code_cccccccccccccccc.air.json") +} + +// Off by default: with the variable unset the hook returns before touching the workspace, so +// nothing is listed or deleted. Any value that is not a true-like string is off too. +func TestV2GCOffByDefaultDoesNothing(t *testing.T) { + t.Setenv(snapshotGCEnv, "") + assert.False(t, gcEnabled(t.Context())) + + cfg := &runConfig{CodeSource: &codeSourceConfig{Snapshot: &snapshotSourceConfig{}}} + // A nil workspace would panic if the hook got past the gate. + gcAfterSubmit(t.Context(), nil, cfg, "/Workspace/Users/me") +} + +func TestV2GCEnabledByEnv(t *testing.T) { + t.Setenv(snapshotGCEnv, "true") + assert.True(t, gcEnabled(t.Context())) + t.Setenv(snapshotGCEnv, "maybe") + assert.False(t, gcEnabled(t.Context())) +} + +// The time box bounds a run: when listing blocks past the deadline, the run returns and +// deletes nothing, since the references it could not read might keep any object. +func TestV2GCTimeBoxStopsWithoutDeleting(t *testing.T) { + clock := newGCClock() + f := newMemFiler(clock) + seedObject(t, f, v2TestID(10)) + clock.Advance(v2GCAge + time.Hour) + f.blockDir = v2ObjectsDir + + ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond) + defer cancel() + start := time.Now() + runV2GC(ctx, f, clock.Now()) + assert.Less(t, time.Since(start), 2*time.Second) + assert.Contains(t, f.names(), v2ObjectName(v2TestID(10))) +} + +// The GC runs at most once per v2GCInterval per store: a run within the interval of the marker +// does nothing, and one after it runs again. +func TestV2GCMarkerThrottlesRuns(t *testing.T) { + clock := newGCClock() + f := newMemFiler(clock) + base := clock.Now() + // seedOrphan writes an object that is already older than A at the base time. + seedOrphan := func(n int) { + clock.now = base.Add(-(v2GCAge + time.Hour)) + seedObject(t, f, v2TestID(n)) + clock.now = base + } + + seedOrphan(11) + runV2GC(t.Context(), f, base) + assert.NotContains(t, f.names(), v2ObjectName(v2TestID(11)), "the first run is due") + + seedOrphan(12) + runV2GC(t.Context(), f, base.Add(time.Hour)) + assert.Contains(t, f.names(), v2ObjectName(v2TestID(12)), "throttled within the interval") + + runV2GC(t.Context(), f, base.Add(v2GCInterval+time.Hour)) + assert.NotContains(t, f.names(), v2ObjectName(v2TestID(12)), "runs again after the interval") +} + +// A submit reuses an object younger than R and writes a full object again once it is older. +// The writer's reuse rule is the other half of the safety margin. +func TestV2WriterReusesOnlyFreshObjects(t *testing.T) { + clock := newGCClock() + f := newMemFiler(clock) + repo := writeV2Tree(t, v2TestTree(30)) + files, err := snapshotFiles(t.Context(), repo, nil, false) + require.NoError(t, err) + uploadPath := "/Workspace/x/.internal" + + var result snapshotResult + require.NoError(t, uploadSnapshotV2(t.Context(), f, uploadPath, repo, files, &result, clock.Now())) + objectWrites := func() int { + n := 0 + for name, c := range f.writes { + if strings.HasSuffix(name, ".tar.gz") { + n += c + } + } + return n + } + first := objectWrites() + require.Positive(t, first) + + // Drop the manifest so the next submit takes the full path and considers each object. + require.NoError(t, f.Delete(t.Context(), path.Base(result.CodeSourcePath))) + clock.Advance(v2ReuseWindow - time.Hour) + require.NoError(t, uploadSnapshotV2(t.Context(), f, uploadPath, repo, files, &result, clock.Now())) + assert.Equal(t, first, objectWrites(), "fresh objects are reused") + + require.NoError(t, f.Delete(t.Context(), path.Base(result.CodeSourcePath))) + clock.Advance(2 * time.Hour) + require.NoError(t, uploadSnapshotV2(t.Context(), f, uploadPath, repo, files, &result, clock.Now())) + assert.Equal(t, 2*first, objectWrites(), "objects older than R are written again") +} + +// A rerun of an unchanged snapshot renews the manifest's mtime, which is its lease. +func TestV2RenewalRefreshesManifestMtime(t *testing.T) { + clock := newGCClock() + f := newMemFiler(clock) + repo := writeV2Tree(t, v2TestTree(30)) + files, err := snapshotFiles(t.Context(), repo, nil, false) + require.NoError(t, err) + uploadPath := "/Workspace/x/.internal" + + var result snapshotResult + require.NoError(t, uploadSnapshotV2(t.Context(), f, uploadPath, repo, files, &result, clock.Now())) + name := path.Base(result.CodeSourcePath) + + clock.Advance(20 * 24 * time.Hour) + require.NoError(t, uploadSnapshotV2(t.Context(), f, uploadPath, repo, files, &result, clock.Now())) + info, err := f.Stat(t.Context(), name) + require.NoError(t, err) + assert.True(t, info.ModTime().Equal(clock.Now()), "manifest mtime %v, want %v", info.ModTime(), clock.Now()) +} + +// An unchanged rerun whose object is missing rewrites it, rather than renewing a manifest that +// points at nothing. +func TestV2RenewalRepairsMissingObject(t *testing.T) { + clock := newGCClock() + f := newMemFiler(clock) + repo := writeV2Tree(t, v2TestTree(30)) + files, err := snapshotFiles(t.Context(), repo, nil, false) + require.NoError(t, err) + uploadPath := "/Workspace/x/.internal" + + var result snapshotResult + require.NoError(t, uploadSnapshotV2(t.Context(), f, uploadPath, repo, files, &result, clock.Now())) + var manifest v2Manifest + rc, err := f.Read(t.Context(), path.Base(result.CodeSourcePath)) + require.NoError(t, err) + require.NoError(t, json.NewDecoder(rc).Decode(&manifest)) + rc.Close() + victim := v2ObjectName(manifest.Members[0].ID) + require.NoError(t, f.Delete(t.Context(), victim)) + + require.NoError(t, uploadSnapshotV2(t.Context(), f, uploadPath, repo, files, &result, clock.Now())) + assert.Contains(t, f.names(), victim) + // Written once by the first submit and once more to repair it. + assert.Equal(t, 2, f.writes[path.Base(victim)]) +} diff --git a/cmd/air/snapshot_v2_test.go b/cmd/air/snapshot_v2_test.go index 4ca4e15986a..8ffd41f37ba 100644 --- a/cmd/air/snapshot_v2_test.go +++ b/cmd/air/snapshot_v2_test.go @@ -205,7 +205,7 @@ func TestV2ManifestSchema(t *testing.T) { files, err := snapshotFiles(t.Context(), repo, nil, false) require.NoError(t, err) var result snapshotResult - require.NoError(t, uploadSnapshotV2(t.Context(), store.f, testSnapshotArtifactPath+"/.internal", repo, files, &result)) + require.NoError(t, uploadSnapshotV2(t.Context(), store.f, testSnapshotArtifactPath+"/.internal", repo, files, &result, time.Now())) rc, err := store.f.Read(t.Context(), path.Base(result.CodeSourcePath)) require.NoError(t, err) @@ -439,7 +439,7 @@ func TestV2InterruptedUploadLeavesNoManifest(t *testing.T) { store := newV2Store(t, func(name string, _ int) bool { return name == manifestName }) uploadPath := testSnapshotArtifactPath + "/.internal" var result snapshotResult - require.Error(t, uploadSnapshotV2(t.Context(), store.f, uploadPath, repo, files, &result)) + require.Error(t, uploadSnapshotV2(t.Context(), store.f, uploadPath, repo, files, &result, time.Now())) _, err = store.f.Stat(t.Context(), manifestName) assert.ErrorIs(t, err, os.ErrNotExist) assert.Positive(t, store.totalWrites(), "the objects were written before the manifest failed") @@ -448,7 +448,7 @@ func TestV2InterruptedUploadLeavesNoManifest(t *testing.T) { store.setFailWrite(nil) store.resetWrites() result = snapshotResult{} - require.NoError(t, uploadSnapshotV2(t.Context(), store.f, uploadPath, repo, files, &result)) + require.NoError(t, uploadSnapshotV2(t.Context(), store.f, uploadPath, repo, files, &result, time.Now())) assert.Equal(t, 1, store.totalWrites(), "only the manifest is written on retry") assert.Equal(t, 1, store.written(manifestName)) } @@ -463,7 +463,7 @@ func TestV2RetriesTransientWriteFailure(t *testing.T) { } store := newV2Store(t, failFirst) var result snapshotResult - require.NoError(t, uploadSnapshotV2(t.Context(), store.f, testSnapshotArtifactPath+"/.internal", repo, files, &result)) + require.NoError(t, uploadSnapshotV2(t.Context(), store.f, testSnapshotArtifactPath+"/.internal", repo, files, &result, time.Now())) // Every object's first write failed, so each was retried and the upload still succeeded. assert.Positive(t, store.failed.Load()) } @@ -483,7 +483,7 @@ func TestV2UploadConcurrencyIsBounded(t *testing.T) { list[i] = v2Member{ID: hex.EncodeToString(sum), TarSize: 1024, fragment: fragment} } store := newV2Store(t, nil) - require.NoError(t, putV2Members(t.Context(), store.f, list, tmp)) + require.NoError(t, putV2Members(t.Context(), store.f, list, tmp, time.Now())) assert.LessOrEqual(t, int(store.maxSeen.Load()), v2UploadConcurrency) assert.Greater(t, int(store.maxSeen.Load()), 1, "uploads should overlap") @@ -501,15 +501,18 @@ func TestV2UnchangedSnapshotWritesNothing(t *testing.T) { require.NoError(t, err) var first snapshotResult - require.NoError(t, uploadSnapshotV2(t.Context(), store.f, uploadPath, repo, files, &first)) + require.NoError(t, uploadSnapshotV2(t.Context(), store.f, uploadPath, repo, files, &first, time.Now())) before := store.totalWrites() assert.Positive(t, before) var second snapshotResult - require.NoError(t, uploadSnapshotV2(t.Context(), store.f, uploadPath, repo, files, &second)) + require.NoError(t, uploadSnapshotV2(t.Context(), store.f, uploadPath, repo, files, &second, time.Now())) assert.Equal(t, first.CodeSourcePath, second.CodeSourcePath) - assert.Equal(t, before, store.totalWrites(), "unchanged resubmit writes nothing") + // The only write is the manifest rewrite that renews its lease. No object is written. + assert.Equal(t, before+1, store.totalWrites(), "unchanged resubmit renews the manifest and nothing else") + assert.Equal(t, 2, store.written(path.Base(second.CodeSourcePath)), "written once by each submit") assert.Equal(t, new(int64(0)), second.UploadDurationMs) + assert.Equal(t, first.SizeBytes, second.SizeBytes) } // The manifest records the K it was written with, and the next snapshot of the same @@ -532,7 +535,7 @@ func TestV2KRatchetsFromPreviousManifest(t *testing.T) { files, err := snapshotFiles(t.Context(), repo, nil, false) require.NoError(t, err) var result snapshotResult - require.NoError(t, uploadSnapshotV2(t.Context(), store.f, uploadPath, repo, files, &result)) + require.NoError(t, uploadSnapshotV2(t.Context(), store.f, uploadPath, repo, files, &result, time.Now())) rc, err := store.f.Read(t.Context(), path.Base(result.CodeSourcePath)) require.NoError(t, err) diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index c48ebb04df2..a2ee717231b 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -1309,6 +1309,8 @@ func (s *FakeWorkspace) importFile(filePath string, body []byte, overwrite bool, ObjectType: "FILE", Path: filePath, ObjectId: nextTreenodeID(), + // The workspace reports when a file was last written; callers rely on it for age. + ModifiedAt: time.Now().UnixMilli(), }, Data: body, }