Compose VM rootfs from shared layer blobs

lib/images/compose.go

@@ -0,0 +1,55 @@
+package images
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+)
+
+// composeRootfs validates the persisted model and merges its layers into
+// dest in manifest order, reading each layer blob from the shared OCI cache.
+// Whiteout and opaque-directory markers are interpreted as each layer is
+// applied. Any previous tree at dest is replaced: callers must not read dest
+// concurrently, and a failure between the remove and the rename leaves dest
+// absent. The export root is always 0755 regardless of the last layer's tar
+// root entry, matching the mode the previous unpack path created. A crash
+// can also strand .compose-* staging directories in dest's parent, the same
+// way .unpack-* directories can strand under layer builds.
+func (c *ociClient) composeRootfs(ctx context.Context, dest, layoutTag string, model *imageManifestModel) error {
+	if err := validateManifestModel(layoutTag, model); err != nil {
+		return fmt.Errorf("validate manifest model: %w", err)
+	}
+	parent := filepath.Dir(dest)
+	if err := os.MkdirAll(parent, 0755); err != nil {
+		return fmt.Errorf("create compose parent: %w", err)
+	}
+	staging, err := os.MkdirTemp(parent, ".compose-*")
+	if err != nil {
+		return fmt.Errorf("create compose directory: %w", err)
+	}
+	defer func() {
+		if err := removePath(staging); err != nil {
+			slog.Warn("failed to remove compose staging directory", "dir", staging, "error", err)
+		}
+	}()
+
+	for i, desc := range model.Layers {
+		if _, err := unpackCachedLayer(ctx, c.cacheBlobDir(), desc, staging, composeOnDiskFormat()); err != nil {
+			return fmt.Errorf("apply layer %d: %w", i, err)
+		}
+	}
+	// The export directory must stay traversable by other readers; MkdirTemp
+	// creates it 0700.
+	if err := os.Chmod(staging, 0755); err != nil {
+		return fmt.Errorf("set compose directory mode: %w", err)
+	}
+	if err := removePath(dest); err != nil {
+		return fmt.Errorf("replace compose directory: %w", err)
+	}
+	if err := os.Rename(staging, dest); err != nil {
+		return fmt.Errorf("install compose directory: %w", err)
+	}
+	return nil
+}

lib/images/compose_test.go

@@ -0,0 +1,215 @@
+package images
+
+import (
+	"context"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	gcr "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/empty"
+	"github.com/google/go-containerregistry/pkg/v1/mutate"
+	"github.com/kernel/hypeman/lib/paths"
+	"github.com/stretchr/testify/require"
+)
+
+// composeTestImage builds the standard two-layer fixture: a base layer with
+// content the top layer deletes, masks, replaces, and extends.
+func composeTestImage(t *testing.T) gcr.Image {
+	t.Helper()
+
+	base := specLayer(t, []tarEntrySpec{
+		{name: "etc/", isDir: true, mode: 0755},
+		{name: "etc/config.txt", content: "original", mode: 0644},
+		{name: "app/", isDir: true, mode: 0755},
+		{name: "app/main.txt", content: "v1", mode: 0644},
+		{name: "data/", isDir: true, mode: 0755},
+		{name: "data/old.txt", content: "stale", mode: 0644},
+		{name: "replacedir/", isDir: true, mode: 0755},
+		{name: "replacedir/inner.txt", content: "inner", mode: 0644},
+	})
+	top := specLayer(t, []tarEntrySpec{
+		{name: "etc/.wh.config.txt", content: "", mode: 0644},
+		{name: "app/main.txt", content: "v2", mode: 0644},
+		{name: "data/.wh..wh..opq", content: "", mode: 0644},
+		{name: "data/new.txt", content: "new", mode: 0644},
+		{name: "bin/", isDir: true, mode: 0755},
+		{name: "bin/tool", content: "tool", mode: 0755},
+		{name: "replacedir", content: "now a file", mode: 0644},
+	})
+
+	img, err := mutate.AppendLayers(empty.Image, base, top)
+	require.NoError(t, err)
+	return img
+}
+
+// composeFixture composes the standard fixture image into the shared OCI cache
+// and returns a client plus its validated manifest model.
+func composeFixture(t *testing.T, p *paths.Paths) (*ociClient, string, *imageManifestModel) {
+	t.Helper()
+
+	img := composeTestImage(t)
+	writeLayerTestLayout(t, p, img)
+
+	client, err := newOCIClient(p.SystemOCICache())
+	require.NoError(t, err)
+	digest, err := img.Digest()
+	require.NoError(t, err)
+	tag := digestToLayoutTag(digest.String())
+	bundle, err := client.extractOCIImageBundle(tag)
+	require.NoError(t, err)
+	return client, tag, bundle.Model
+}
+
+func TestComposeRootfsWhiteoutsAndOrdering(t *testing.T) {
+	p := paths.New(t.TempDir())
+	client, tag, model := composeFixture(t, p)
+	require.Len(t, model.Layers, 2)
+
+	dest := filepath.Join(t.TempDir(), "rootfs")
+	// Pre-populate dest and weaken its mode so composition must replace the
+	// whole tree and restore the 0755 export-directory mode.
+	require.NoError(t, os.MkdirAll(filepath.Join(dest, "junk"), 0700))
+	require.NoError(t, os.WriteFile(filepath.Join(dest, "junk", "stale.txt"), []byte("stale"), 0644))
+	require.NoError(t, client.composeRootfs(context.Background(), dest, tag, model))
+
+	// Stale content is gone and the export directory mode is restored.
+	_, err := os.Lstat(filepath.Join(dest, "junk"))
+	require.True(t, os.IsNotExist(err), "composition must replace the previous tree")
+	info, err := os.Stat(dest)
+	require.NoError(t, err)
+	require.Equal(t, os.FileMode(0755), info.Mode().Perm())
+
+	// Whiteout removed the base entry.
+	_, err = os.Lstat(filepath.Join(dest, "etc", "config.txt"))
+	require.True(t, os.IsNotExist(err), "whiteout must delete the base entry")
+
+	// Plain replacement.
+	data, err := os.ReadFile(filepath.Join(dest, "app", "main.txt"))
+	require.NoError(t, err)
+	require.Equal(t, "v2", string(data))
+
+	// Opaque directory masked the base content.
+	_, err = os.Lstat(filepath.Join(dest, "data", "old.txt"))
+	require.True(t, os.IsNotExist(err), "opaque marker must mask base contents")
+	data, err = os.ReadFile(filepath.Join(dest, "data", "new.txt"))
+	require.NoError(t, err)
+	require.Equal(t, "new", string(data))
+
+	// Directory replaced by a regular file.
+	info, err = os.Lstat(filepath.Join(dest, "replacedir"))
+	require.NoError(t, err)
+	require.False(t, info.IsDir())
+	data, err = os.ReadFile(filepath.Join(dest, "replacedir"))
+	require.NoError(t, err)
+	require.Equal(t, "now a file", string(data))
+
+	// New entry present with its mode.
+	info, err = os.Stat(filepath.Join(dest, "bin", "tool"))
+	require.NoError(t, err)
+	require.Equal(t, os.FileMode(0755), info.Mode().Perm())
+
+	// No whiteout markers survive composition.
+	require.NoError(t, filepath.Walk(dest, func(path string, info os.FileInfo, err error) error {
+		require.NoError(t, err)
+		require.NotContains(t, info.Name(), whiteoutPrefix, "whiteout marker leaked into composed rootfs")
+		return nil
+	}))
+}
+
+func TestComposeRootfsEmptyLayers(t *testing.T) {
+	p := paths.New(t.TempDir())
+	client, err := newOCIClient(p.SystemOCICache())
+	require.NoError(t, err)
+	model := &imageManifestModel{
+		SchemaVersion: manifestModelSchemaVersion,
+		Digest:        "sha256:" + strings.Repeat("ab", 32),
+		RootFSType:    "layers",
+		Config: manifestConfigRef{
+			Digest:    "sha256:" + strings.Repeat("cd", 32),
+			MediaType: "application/vnd.oci.image.config.v1+json",
+		},
+		Layers: make([]layerDescriptor, 0),
+	}
+	dest := filepath.Join(t.TempDir(), "rootfs")
+	require.NoError(t, client.composeRootfs(context.Background(), dest, model.Digest, model))
+	entries, err := os.ReadDir(dest)
+	require.NoError(t, err)
+	require.Empty(t, entries)
+}
+
+func TestComposeRootfsInvalidModel(t *testing.T) {
+	p := paths.New(t.TempDir())
+	client, tag, model := composeFixture(t, p)
+
+	model.Config.DiffIDs = model.Config.DiffIDs[:1]
+	err := client.composeRootfs(context.Background(), filepath.Join(t.TempDir(), "rootfs"), tag, model)
+	require.ErrorContains(t, err, "config rootfs.diff_ids has 1 entries but manifest has 2 layers")
+}
+
+func TestComposeRootfsMissingBlob(t *testing.T) {
+	p := paths.New(t.TempDir())
+	client, err := newOCIClient(p.SystemOCICache())
+	require.NoError(t, err)
+
+	digestHex := "sha256:" + strings.Repeat("ab", 32)
+	model := &imageManifestModel{
+		SchemaVersion: manifestModelSchemaVersion,
+		Digest:        digestHex,
+		RootFSType:    "layers",
+		Config: manifestConfigRef{
+			Digest:    "sha256:" + strings.Repeat("cd", 32),
+			MediaType: "application/vnd.oci.image.config.v1+json",
+			DiffIDs:   []string{"sha256:" + strings.Repeat("ef", 32)},
+		},
+		Layers: []layerDescriptor{{
+			Digest:    "sha256:" + strings.Repeat("01", 32),
+			MediaType: "application/vnd.oci.image.layer.v1.tar+gzip",
+			DiffID:    "sha256:" + strings.Repeat("ef", 32),
+		}},
+	}
+	err = client.composeRootfs(context.Background(), t.TempDir(), digestHex, model)
+	require.ErrorContains(t, err, "missing from oci cache")
+}
+
+func TestComposeRootfsDiffIDMismatch(t *testing.T) {
+	p := paths.New(t.TempDir())
+	client, tag, model := composeFixture(t, p)
+
+	// Desynchronize the top layer's diff id from its content while keeping the
+	// model internally consistent, so validation passes and the mismatch is
+	// caught against the unpacked stream instead.
+	forged := "sha256:" + strings.Repeat("ff", 32)
+	model.Layers[1].DiffID = forged
+	model.Config.DiffIDs[1] = forged
+
+	err := client.composeRootfs(context.Background(), filepath.Join(t.TempDir(), "rootfs"), tag, model)
+	require.ErrorContains(t, err, "diff id mismatch")
+}
+
+// TestComposeRootfsExportsValidErofs composes the fixture image and exports it
+// to erofs, then verifies the filesystem passes fsck.
+func TestComposeRootfsExportsValidErofs(t *testing.T) {
+	if _, err := exec.LookPath("mkfs.erofs"); err != nil {
+		t.Skip("mkfs.erofs not available")
+	}
+	if _, err := exec.LookPath("fsck.erofs"); err != nil {
+		t.Skip("fsck.erofs not available")
+	}
+
+	p := paths.New(t.TempDir())
+	client, tag, model := composeFixture(t, p)
+
+	dest := filepath.Join(t.TempDir(), "rootfs")
+	require.NoError(t, client.composeRootfs(context.Background(), dest, tag, model))
+
+	diskPath := filepath.Join(t.TempDir(), "rootfs.erofs")
+	size, err := ExportRootfs(dest, diskPath, FormatErofs)
+	require.NoError(t, err)
+	require.Greater(t, size, int64(0))
+
+	output, err := exec.Command("fsck.erofs", "--extract", diskPath).CombinedOutput()
+	require.NoError(t, err, "fsck.erofs failed: %s", output)
+}

lib/images/layer_artifact.go

@@ -94,9 +94,8 @@ func layerArtifactRecordPath(p *paths.Paths, layerHex string) string {
 
 // layerMapOptions preserves tar ownership when running as root. Otherwise
 // umoci's rootless mode skips chown and stands in empty files for device nodes.
-// Unlike unpackLayers in oci.go, which maps container root to the current
-// user, this deliberately leaves ownership untouched as root: artifacts must
-// keep the layer's on-disk ownership for later stacking.
+// As root this deliberately leaves ownership untouched: artifacts must keep
+// the layer's on-disk ownership for later stacking.
 func layerMapOptions() layer.MapOptions {
 	return layer.MapOptions{Rootless: os.Geteuid() != 0}
 }
@@ -109,6 +108,14 @@ func layerArtifactOnDiskFormat() layer.OnDiskFormat {
 	return layer.OverlayfsRootfs{MapOptions: layerMapOptions()}
 }
 
+// composeOnDiskFormat applies whiteouts against the tree being composed:
+// deletions execute immediately on the destination instead of becoming
+// overlayfs whiteout inodes, since the composed tree is mounted as a single
+// lower filesystem rather than stacked.
+func composeOnDiskFormat() layer.OnDiskFormat {
+	return layer.DirRootfs{MapOptions: layerMapOptions()}
+}
+
 // readLayerRecord loads the artifact record for a layer digest, if present.
 // A missing record returns (nil, nil): the layer simply was never
 // materialized.
@@ -215,7 +222,7 @@ func (m *manager) materializeLayerArtifactOnce(ctx context.Context, desc layerDe
 		}
 	}()
 
-	stats, err := unpackCachedLayer(ctx, m.paths, desc, unpackDir, layerArtifactOnDiskFormat())
+	stats, err := unpackCachedLayer(ctx, m.paths.OCICacheBlobDir(), desc, unpackDir, layerArtifactOnDiskFormat())
 	if err != nil {
 		return nil, err
 	}
@@ -274,11 +281,12 @@ func (r contextReader) Read(p []byte) (int, error) {
 	return r.reader.Read(p)
 }
 
-// unpackCachedLayer locates desc's blob in the shared OCI cache, unpacks it
-// into dest, and verifies both the blob digest and the diff ID when the
-// descriptor carries one. The caller must have validated desc.Digest.
-func unpackCachedLayer(ctx context.Context, p *paths.Paths, desc layerDescriptor, dest string, onDisk layer.OnDiskFormat) (*unpackStats, error) {
-	blobPath := p.OCICacheBlob(strings.TrimPrefix(desc.Digest, "sha256:"))
+// unpackCachedLayer locates desc's blob under blobDir (the OCI layout's
+// blobs/sha256 directory), unpacks it into dest, and verifies both the blob
+// digest and the diff ID when the descriptor carries one. The caller must
+// have validated desc.Digest.
+func unpackCachedLayer(ctx context.Context, blobDir string, desc layerDescriptor, dest string, onDisk layer.OnDiskFormat) (*unpackStats, error) {
+	blobPath := filepath.Join(blobDir, strings.TrimPrefix(desc.Digest, "sha256:"))
 	if _, err := os.Stat(blobPath); err != nil {
 		if os.IsNotExist(err) {
 			return nil, fmt.Errorf("layer blob missing from oci cache: %s", desc.Digest)

lib/images/layer_artifact_test.go

@@ -19,7 +19,6 @@ import (
 	"github.com/google/go-containerregistry/pkg/v1/layout"
 	"github.com/google/go-containerregistry/pkg/v1/mutate"
 	"github.com/kernel/hypeman/lib/paths"
-	"github.com/opencontainers/umoci/oci/layer"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/sys/unix"
 )
@@ -31,12 +30,6 @@ import (
 // later be stacked.
 const whiteoutPrefix = ".wh."
 
-// composeOnDiskFormat applies whiteouts against the tree being composed. It
-// belongs to the composition flow and moves to production with that change.
-func composeOnDiskFormat() layer.OnDiskFormat {
-	return layer.DirRootfs{MapOptions: layerMapOptions()}
-}
-
 const testTarGzMediaType = "application/vnd.oci.image.layer.v1.tar+gzip"
 
 // writeLayerTestLayout writes img into the shared OCI cache of p tagged with

lib/images/manager.go

@@ -451,7 +451,15 @@ func (m *manager) newPendingImageMetadata(ref *ResolvedRef, req CreateImageReque
 func (m *manager) buildImage(ctx context.Context, ref *ResolvedRef, credentials *authn.AuthConfig, buildID string) {
 	buildStart := time.Now()
 	buildStatus := "failed"
-	buildDir := m.paths.SystemBuild(ref.String())
+	// Key the build directory by digest so two pending builds of the same
+	// ref with different digests never compose into (and delete) the same
+	// rootfs. This matches the queue's digest-based deduplication.
+	digestHex := ref.DigestHex()
+	if digestHex == "" {
+		m.updateStatusByDigest(ref, StatusFailed, fmt.Errorf("missing resolved digest"), buildID)
+		return
+	}
+	buildDir := m.paths.SystemBuild(digestHex)
 	tempDir := filepath.Join(buildDir, "rootfs")
 	defer func() {
 		m.recordBuildMetrics(ctx, buildStart, buildStatus)