diff --git a/.env.example b/.env.example index d18c7324..67d62708 100644 --- a/.env.example +++ b/.env.example @@ -464,3 +464,25 @@ ICAPTCHA_PUBKEY= ICAPTCHA_REQUIRED_LEVEL=3 # Base URL of the iCaptcha service. ICAPTCHA_URL=https://icaptcha.gitlawb.com + +# ── Model hosting (experimental, off by default) ────────────────────────── +# Signed model releases on public repos, with weights stored as +# content-addressed blocks outside git. Off by default; not yet usable end to +# end: this ships the storage and schema only, no publish or read routes yet. +TWIGPINE_MODEL_HOSTING_ENABLED=false +# Per-identity quota for claimed model blocks, in bytes (default 256 GiB). +TWIGPINE_MODEL_QUOTA_BYTES=274877906944 +# Total model bytes this node will hold, claimed plus open sessions (1 TiB). +TWIGPINE_MODEL_NODE_QUOTA_BYTES=1099511627776 +# Seconds an uploaded block may sit unclaimed before the sweep deletes it. +TWIGPINE_MODEL_UNCLAIMED_TTL_SECS=86400 +# Base lifetime of an upload session; each accepted block renews it. +TWIGPINE_MODEL_SESSION_TTL_SECS=43200 +# Optional cap on the sum of one identity's open upload sessions, in bytes. +# Unset means "use TWIGPINE_MODEL_QUOTA_BYTES". +#TWIGPINE_MODEL_MAX_SESSION_BYTES= +# Where model blocks live. Defaults to a model-blocks dir beside the data dir. +# Leave the bucket empty to use the filesystem; set it to store blocks in S3 +# using the same AWS_* endpoint/credentials as the Tigris repo backend. +TWIGPINE_BLOCKSTORE_DIR=./data/model-blocks +TWIGPINE_BLOCKSTORE_S3_BUCKET= diff --git a/Cargo.lock b/Cargo.lock index f8b87889..4dcc8899 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2208,7 +2208,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] @@ -2701,7 +2701,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ab67060fc6b8ef687992d439ca0fa36e7ed17e9a0b16b25b601e8757df720de" dependencies = [ "data-encoding", - "syn 1.0.109", + "syn 2.0.117", ] [[package]] @@ -3141,6 +3141,12 @@ dependencies = [ "siphasher", ] +[[package]] +name = "fastcdc" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0431c3132902a1bed6fe77dcfa5d03844e083f263b81288246a2d43dd7d4b2ad" + [[package]] name = "fastrand" version = "2.3.0" @@ -7458,6 +7464,7 @@ dependencies = [ "crypto_box", "curve25519-dalek 4.1.3", "ed25519-dalek 2.2.0", + "fastcdc", "hex", "multibase", "multihash", @@ -7960,7 +7967,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/README.md b/README.md index 107cc23b..16508f96 100644 --- a/README.md +++ b/README.md @@ -431,6 +431,7 @@ Important node settings: | `TWIGPINE_AUTO_SYNC` | Enable automatic sync from known peers. | | `TWIGPINE_RECONCILIATION_SWEEP` | Enable the hourly durability sweep that re-pins/backstops missing objects (default `true`; disabled when no IPFS/Pinata backend is configured). Public pin repair runs against any configured backend (IPFS, Pinata, or both). Encrypted recovery repair requires local IPFS (`TWIGPINE_IPFS_API`); Pinata-only nodes reconcile public pins only. | | `TWIGPINE_MAX_PACK_BYTES` | Max git pack body size for smart-HTTP routes. | +| `TWIGPINE_MODEL_HOSTING_ENABLED` | Enable experimental model hosting (signed releases, block storage). Off by default; see `docs/RUN-A-NODE.md` for the quota and block-store knobs. | | `TWIGPINE_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, or `info/refs` advertisement may run before it is aborted (504). Default 600. Also bounds the withheld-blob classification walk (on both the upload-pack serve and receive-pack replication paths) and the push-side pin-candidate discovery (`rev-list` / `cat-file`), each reaped via process-group teardown at the deadline. On the path-scoped upload-pack path the classification walk and the pack serve share ONE deadline, so this value bounds their combined duration rather than granting each stage a full budget: a walk that consumes it leaves the serve nothing and the clone gets a 504. Serving large path-scoped repos may therefore need a higher value than they did when each stage was budgeted separately. Accepted range is 1 to 3153600000 (100 years), since the node derives deadlines from this value and a larger one cannot be represented. | | `TWIGPINE_GIT_ACQUIRE_TIMEOUT_SECS` | Max seconds the storage-acquisition phase (Tigris HEAD/GET, push advisory-lock) of a served git op may run before the request is shed with a 503, separate from the git-run timeout. The concurrency permit is released on expiry so a stalled backend cannot pin the pool. Default 30. | | `TWIGPINE_MAX_CONCURRENT_GIT_OPS` | Max concurrent served git READ ops (upload-pack and its `info/refs` advertisement) across all callers; over-cap sheds a 503 + Retry-After. Anonymous reads draw from this pool, so pair it with `TWIGPINE_MAX_CONCURRENT_READS_PER_CALLER`. Pushes and the receive-pack advertisement have their own pools, so a read flood cannot shed an authenticated push. Default 128. | diff --git a/crates/twigpine-core/Cargo.toml b/crates/twigpine-core/Cargo.toml index 867e9d11..aebd3f53 100644 --- a/crates/twigpine-core/Cargo.toml +++ b/crates/twigpine-core/Cargo.toml @@ -29,6 +29,11 @@ curve25519-dalek = "4" crypto_box = { version = "0.9", features = ["std", "chacha20"] } chacha20poly1305 = "0.10" +# Content-defined chunking (FastCDC, the 2020 algorithm variant) for the +# model-block format. Streaming over a reader, so memory stays proportional +# to the block size rather than the file size. +fastcdc = "5" + [dev-dependencies] tokio = { workspace = true } # Not optional here: the redirect matrix must run under a bare diff --git a/crates/twigpine-core/src/lib.rs b/crates/twigpine-core/src/lib.rs index d53b5ea2..ba0f19b8 100644 --- a/crates/twigpine-core/src/lib.rs +++ b/crates/twigpine-core/src/lib.rs @@ -8,6 +8,7 @@ pub mod identity; pub mod identity_path; pub mod legacy_env; pub mod legacy_headers; +pub mod model; pub mod node_denial; // `url` is the one dependency here that drags a tail (idna, then the icu // crates), and twigpine-core is allowlisted to stay embeddable. Every client that diff --git a/crates/twigpine-core/src/model/chunking.rs b/crates/twigpine-core/src/model/chunking.rs new file mode 100644 index 00000000..190c1e58 --- /dev/null +++ b/crates/twigpine-core/src/model/chunking.rs @@ -0,0 +1,139 @@ +//! Content-defined chunking and block packing. +//! +//! Chunking uses FastCDC (the 2020 algorithm variant) at 8 / 64 / 128 KiB +//! min / avg / max. Packing into blocks lives in `reconstruction::pack_file`, +//! which seals at the 64 MiB target or end of stream. + +use std::io::Read; + +use fastcdc::v2020::StreamCDC; + +use super::sha256_hex; + +pub const CHUNK_MIN: usize = 8 * 1024; +pub const CHUNK_AVG: usize = 64 * 1024; +pub const CHUNK_MAX: usize = 128 * 1024; +pub const BLOCK_TARGET: usize = 64 * 1024 * 1024; + +/// One content-defined chunk of a file. `data` is present while packing; the +/// id is the chunk's sha256, which is how the chunk dedups across files and +/// releases. +#[derive(Debug)] +pub struct Chunk { + pub sha256: String, + pub len: usize, + pub offset: u64, + pub data: Vec, +} + +/// A sealed block: a run of chunks and its content id. +#[derive(Debug)] +pub struct Block { + pub sha256: String, + pub chunks: Vec, + pub data: Vec, +} + +/// Stream a reader into content-defined chunks. Memory stays bounded by the +/// maximum chunk size, never the file size. +pub fn chunk_reader(reader: R) -> impl Iterator> { + StreamCDC::new(reader, CHUNK_MIN, CHUNK_AVG, CHUNK_MAX).map(|r| { + let c = r?; + Ok(Chunk { + sha256: sha256_hex(&c.data), + len: c.data.len(), + offset: c.offset, + data: c.data, + }) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::model::gen_bytes; + use std::io::Cursor; + + const SEED: u64 = 0x9e3779b97f4a7c15; + + fn chunk_all(bytes: &[u8]) -> Vec { + chunk_reader(Cursor::new(bytes.to_vec())) + .collect::>() + .unwrap() + } + + #[test] + fn chunking_is_deterministic() { + let bytes = gen_bytes(3 * 1024 * 1024, SEED); + let a = chunk_all(&bytes); + let b = chunk_all(&bytes); + assert_eq!(a.len(), b.len()); + for (x, y) in a.iter().zip(b.iter()) { + assert_eq!(x.sha256, y.sha256); + assert_eq!(x.len, y.len); + assert_eq!(x.offset, y.offset); + } + } + + #[test] + fn chunk_sizes_stay_within_bounds() { + let bytes = gen_bytes(10 * 1024 * 1024, SEED); + let chunks = chunk_all(&bytes); + assert!(chunks.len() > 10); + for (i, c) in chunks.iter().enumerate() { + assert!(c.len <= CHUNK_MAX, "chunk {i} exceeds max"); + if i + 1 < chunks.len() { + // The final chunk may end before the min when the input runs out. + assert!(c.len >= CHUNK_MIN, "chunk {i} below min"); + } + } + } + + #[test] + fn a_small_edit_changes_only_early_chunks() { + let base = gen_bytes(20 * 1024 * 1024, SEED); + let mut edited = vec![0u8; 100]; + edited.extend_from_slice(&base); + + let base_ids: Vec = chunk_all(&base).iter().map(|c| c.sha256.clone()).collect(); + let edited_ids: Vec = chunk_all(&edited) + .iter() + .map(|c| c.sha256.clone()) + .collect(); + + let base_set: std::collections::HashSet<&String> = base_ids.iter().collect(); + let changed = edited_ids + .iter() + .filter(|id| !base_set.contains(id)) + .count(); + // 100 inserted bytes move the first few boundaries; the vast majority + // of a 20 MiB file still chunks identically. + assert!( + changed <= 4, + "{changed} changed chunks for a 100-byte prefix insert" + ); + } + + #[test] + fn chunk_data_never_buffers_the_file() { + // The contract this format depends on: a reader is consumed a chunk + // at a time. Guard it by chunking a reader that would panic if read + // past its claimed end before the iterator asks for more. + struct Stingy { + left: usize, + } + impl Read for Stingy { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + let n = buf.len().min(self.left); + self.left -= n; + buf[..n].fill(1); + Ok(n) + } + } + let mut it = chunk_reader(Stingy { + left: 3 * CHUNK_MAX, + }); + let first = it.next().unwrap().unwrap(); + assert!(first.len <= CHUNK_MAX); + } +} diff --git a/crates/twigpine-core/src/model/manifest.rs b/crates/twigpine-core/src/model/manifest.rs new file mode 100644 index 00000000..90cac811 --- /dev/null +++ b/crates/twigpine-core/src/model/manifest.rs @@ -0,0 +1,388 @@ +//! The signed model release manifest (`gitlawb/model-release/v1`). +//! +//! A release is a git commit carrying `.twigpine/model-release.json` (these +//! bytes) and `.twigpine/model-release.sig` (a detached Ed25519 signature). +//! The signature covers the domain tag `gitlawb-model-release-sig/v1\n` +//! followed by the exact manifest bytes: no canonicalization, so a signer and +//! a verifier can never drift over JSON re-serialization, and the version tag +//! lives inside the signed bytes the way the ref cert's does. +//! +//! Verification anchors on a DID the caller supplies, never the manifest's +//! `publisher_did` field: a key read from inside the artifact only proves +//! self-consistency, and anyone can mint a keypair and name its DID. The +//! manifest's field is still checked for equality so a manifest naming one +//! publisher cannot pass for another. + +use ed25519_dalek::{Signature, Signer, SigningKey}; +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +use crate::did::Did; + +pub const MANIFEST_SCHEMA_V1: &str = "gitlawb/model-release/v1"; + +/// Domain separation tag prepended to the manifest bytes before signing, so +/// a release signature cannot be replayed as a signature over the same bytes +/// in some other protocol. +pub const SIGNING_DOMAIN: &[u8] = b"gitlawb-model-release-sig/v1\n"; + +/// The exact bytes a publisher signs: the domain tag, then the manifest +/// bytes verbatim. +pub fn signing_input(manifest_bytes: &[u8]) -> Vec { + let mut v = Vec::with_capacity(SIGNING_DOMAIN.len() + manifest_bytes.len()); + v.extend_from_slice(SIGNING_DOMAIN); + v.extend_from_slice(manifest_bytes); + v +} + +#[derive(Debug, Error)] +pub enum ManifestError { + #[error("unsupported manifest schema {0:?}")] + UnknownSchema(String), + #[error("manifest names publisher {found}, expected {expected}")] + PublisherMismatch { expected: String, found: String }, + #[error("invalid publisher DID: {0}")] + Did(String), + #[error("invalid signature encoding: {0}")] + SignatureEncoding(String), + #[error("signature does not verify against the expected publisher")] + BadSignature, + #[error("json error: {0}")] + Json(#[from] serde_json::Error), +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum FileFormat { + Safetensors, + Gguf, + Pickle, + Text, + Other, +} + +/// Where a manifest-listed file's bytes live. Files under the chunking +/// threshold stay ordinary git blobs; larger ones are reconstructed from +/// content blocks. +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum StorageKind { + Git, + Chunked, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ManifestFile { + pub path: String, + pub size: u64, + pub sha256: String, + pub format: FileFormat, + pub storage: StorageKind, +} + +/// A publisher's claimed parent release. The node records and displays it; +/// it does not prove the weights derive from the parent. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ParentRef { + pub repo: String, + pub commit: String, + pub manifest_sha256: String, +} + +/// Pointer at the reconstruction object (itself stored as a content block). +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ReconstructionRef { + pub sha256: String, + pub size: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct Manifest { + pub schema: String, + /// `"owner/name"` on the node. + pub repo: String, + pub publisher_did: String, + pub license: String, + #[serde(default)] + pub parent: Option, + pub files: Vec, + #[serde(default)] + pub reconstruction: Option, +} + +impl Manifest { + pub fn to_bytes(&self) -> Result, ManifestError> { + Ok(serde_json::to_vec(self)?) + } + + /// Parse only the schema tag, for version dispatch. A manifest whose tag + /// is unknown is refused before any field is trusted. + pub fn parse(bytes: &[u8]) -> Result { + #[derive(Deserialize)] + struct Tag { + schema: String, + } + let tag: Tag = serde_json::from_slice(bytes)?; + match tag.schema.as_str() { + MANIFEST_SCHEMA_V1 => Ok(serde_json::from_slice(bytes)?), + other => Err(ManifestError::UnknownSchema(other.to_string())), + } + } +} + +/// Sign `manifest_bytes` (already serialized) and return the detached +/// signature as base64 standard, no trailing newline: the exact contents of +/// the `.sig` file. +pub fn sign(key: &SigningKey, manifest_bytes: &[u8]) -> String { + use base64::{engine::general_purpose::STANDARD, Engine}; + let sig = key.sign(&signing_input(manifest_bytes)); + STANDARD.encode(sig.to_bytes()) +} + +/// Verify `sig_text` (the `.sig` file's contents) over `manifest_bytes` +/// against `expected_did`, a DID the caller resolved from outside the +/// manifest. Fails when the manifest names a different publisher, when the +/// schema version is unknown, and when the signature does not verify. +/// Returns the parsed manifest on success. +pub fn verify( + manifest_bytes: &[u8], + sig_text: &str, + expected_did: &Did, +) -> Result { + let manifest = Manifest::parse(manifest_bytes)?; + if manifest.publisher_did != expected_did.to_string() { + return Err(ManifestError::PublisherMismatch { + expected: expected_did.to_string(), + found: manifest.publisher_did, + }); + } + // The named publisher was just proven equal to the caller-supplied DID, + // so the key comes from that DID rather than a second parse of the field. + let key = expected_did + .to_verifying_key() + .map_err(|e| ManifestError::Did(e.to_string()))?; + + use base64::{engine::general_purpose::STANDARD, Engine}; + let sig_bytes = STANDARD + .decode(sig_text.trim()) + .map_err(|e| ManifestError::SignatureEncoding(e.to_string()))?; + let sig_array: [u8; 64] = sig_bytes + .try_into() + .map_err(|_| ManifestError::SignatureEncoding("signature must be 64 bytes".into()))?; + let sig = Signature::from_bytes(&sig_array); + // Strict verification, the same posture the attestation verifier takes. + key.verify_strict(&signing_input(manifest_bytes), &sig) + .map_err(|_| ManifestError::BadSignature)?; + Ok(manifest) +} + +/// Classify a file's format from its leading bytes and path. Honest +/// detection, not a security gate: pickle is flagged so consumers can be +/// warned, never refused. +pub fn detect_format(first_bytes: &[u8], path: &str) -> FileFormat { + if first_bytes.starts_with(b"GGUF") { + return FileFormat::Gguf; + } + // Raw pickle: the protocol opcode 0x80 followed by the version byte. + if first_bytes.len() >= 2 && first_bytes[0] == 0x80 && (2..=5).contains(&first_bytes[1]) { + return FileFormat::Pickle; + } + let lower = path.to_ascii_lowercase(); + let ext = lower.rsplit('.').next().unwrap_or(""); + // Torch/HF checkpoints are zip archives containing pickled tensors. + if first_bytes.starts_with(b"PK\x03\x04") && matches!(ext, "bin" | "pt" | "pth" | "ckpt") { + return FileFormat::Pickle; + } + // safetensors: an 8-byte little-endian header length followed by '{'. + if first_bytes.len() >= 9 + && u64::from_le_bytes(first_bytes[..8].try_into().unwrap()) > 0 + && first_bytes[8] == b'{' + { + return FileFormat::Safetensors; + } + if matches!( + ext, + // `tokenizer.model` is deliberately absent: it is a sentencepiece + // protobuf, so it falls through to `other` like the binaries. + "json" | "txt" | "md" | "py" | "jinja" | "yaml" | "yml" + ) { + return FileFormat::Text; + } + FileFormat::Other +} + +#[cfg(test)] +mod tests { + use super::*; + + fn keypair(seed: u8) -> (SigningKey, Did) { + let key = SigningKey::from_bytes(&[seed; 32]); + let did = Did::from_verifying_key(&key.verifying_key()); + (key, did) + } + + fn sample_manifest(did: &Did) -> Manifest { + Manifest { + schema: MANIFEST_SCHEMA_V1.to_string(), + repo: "alice/llm".to_string(), + publisher_did: did.to_string(), + license: "apache-2.0".to_string(), + parent: None, + files: vec![ + ManifestFile { + path: "model.safetensors".into(), + size: 1024, + sha256: "a".repeat(64), + format: FileFormat::Safetensors, + storage: StorageKind::Chunked, + }, + ManifestFile { + path: "config.json".into(), + size: 256, + sha256: "b".repeat(64), + format: FileFormat::Text, + storage: StorageKind::Git, + }, + ], + reconstruction: Some(ReconstructionRef { + sha256: "c".repeat(64), + size: 512, + }), + } + } + + #[test] + fn a_manifest_signed_by_the_owner_verifies() { + let (key, did) = keypair(1); + let bytes = sample_manifest(&did).to_bytes().unwrap(); + let sig = sign(&key, &bytes); + let parsed = verify(&bytes, &sig, &did).unwrap(); + assert_eq!(parsed.publisher_did, did.to_string()); + assert_eq!(parsed.files.len(), 2); + } + + #[test] + fn a_manifest_naming_another_publisher_fails() { + // Signed by key Y but naming DID X. Verification takes the + // expected DID from the caller, so naming X does not help Y's key. + let (key_y, _did_y) = keypair(2); + let (_key_x, did_x) = keypair(3); + let mut manifest = sample_manifest(&did_x); + manifest.publisher_did = did_x.to_string(); + let bytes = manifest.to_bytes().unwrap(); + let sig = sign(&key_y, &bytes); + // Verified against X's DID: publisher matches, but Y's signature + // does not. + assert!(matches!( + verify(&bytes, &sig, &did_x), + Err(ManifestError::BadSignature) + )); + // Verified against the signer Y: the named publisher X mismatches. + let did_y = Did::from_verifying_key(&key_y.verifying_key()); + assert!(matches!( + verify(&bytes, &sig, &did_y), + Err(ManifestError::PublisherMismatch { .. }) + )); + } + + #[test] + fn a_forged_fully_populated_manifest_fails() { + // A well-formed manifest naming the real owner but signed by a fresh + // keypair must fail: the field is data, not the verifying key. + let (_owner_key, owner_did) = keypair(4); + let (forger, _forger_did) = keypair(5); + let bytes = sample_manifest(&owner_did).to_bytes().unwrap(); + let sig = sign(&forger, &bytes); + assert!(matches!( + verify(&bytes, &sig, &owner_did), + Err(ManifestError::BadSignature) + )); + } + + #[test] + fn one_changed_byte_invalidates_the_signature() { + let (key, did) = keypair(6); + let mut bytes = sample_manifest(&did).to_bytes().unwrap(); + let sig = sign(&key, &bytes); + bytes[10] ^= 1; + assert!(verify(&bytes, &sig, &did).is_err()); + } + + #[test] + fn unknown_schema_is_rejected_before_signature_work() { + let (key, did) = keypair(7); + let mut manifest = sample_manifest(&did); + manifest.schema = "gitlawb/model-release/v99".into(); + let bytes = manifest.to_bytes().unwrap(); + let sig = sign(&key, &bytes); + assert!(matches!( + verify(&bytes, &sig, &did), + Err(ManifestError::UnknownSchema(_)) + )); + } + + #[test] + fn format_detection() { + // safetensors: 8-byte LE header length then '{'. + let mut st = 5u64.to_le_bytes().to_vec(); + st.extend_from_slice(b"{\"a\":0}"); + assert_eq!(detect_format(&st, "m.safetensors"), FileFormat::Safetensors); + assert_eq!(detect_format(b"GGUF\x01rest", "m.gguf"), FileFormat::Gguf); + assert_eq!(detect_format(b"\x80\x04x", "torch.bin"), FileFormat::Pickle); + assert_eq!( + detect_format(b"PK\x03\x04z", "pytorch_model.bin"), + FileFormat::Pickle + ); + // A zip that is NOT a checkpoint extension is not called pickle. + assert_eq!( + detect_format(b"PK\x03\x04z", "archive.zip"), + FileFormat::Other + ); + assert_eq!(detect_format(b"{\"a\":1}", "config.json"), FileFormat::Text); + // Sentencepiece models are protobufs, not text. + assert_eq!(detect_format(b"", "tokenizer.model"), FileFormat::Other); + assert_eq!( + detect_format(b"\x00\x01\x02", "blob.weird"), + FileFormat::Other + ); + } + + /// Regenerate the frozen fixture (run once, deliberately): + /// `cargo test -p twigpine-core manifest_frozen_fixture_write -- --ignored` + #[test] + #[ignore = "writes the fixture files; run only when regenerating"] + fn manifest_frozen_fixture_write() { + let (key, did) = keypair(42); + let bytes = sample_manifest(&did).to_bytes().unwrap(); + let dir = concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/model-release-v1" + ); + std::fs::create_dir_all(dir).unwrap(); + std::fs::write(format!("{dir}/manifest.json"), &bytes).unwrap(); + std::fs::write(format!("{dir}/manifest.sig"), sign(&key, &bytes)).unwrap(); + std::fs::write(format!("{dir}/publisher.did"), did.to_string()).unwrap(); + eprintln!("wrote fixture for {did} to {dir}"); + } + + #[test] + fn the_frozen_v1_fixture_still_verifies() { + let dir = concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/model-release-v1" + ); + let bytes = std::fs::read(format!("{dir}/manifest.json")).unwrap(); + let sig = std::fs::read_to_string(format!("{dir}/manifest.sig")).unwrap(); + let did: Did = std::fs::read_to_string(format!("{dir}/publisher.did")) + .unwrap() + .trim() + .parse() + .unwrap(); + let parsed = verify(&bytes, &sig, &did).unwrap(); + assert_eq!(parsed.repo, "alice/llm"); + // And the frozen signature rejects tampering. + let mut tampered = bytes.clone(); + *tampered.last_mut().unwrap() ^= 1; + assert!(verify(&tampered, &sig, &did).is_err()); + } +} diff --git a/crates/twigpine-core/src/model/mod.rs b/crates/twigpine-core/src/model/mod.rs new file mode 100644 index 00000000..cf969d0a --- /dev/null +++ b/crates/twigpine-core/src/model/mod.rs @@ -0,0 +1,37 @@ +//! Model hosting formats shared by the node and the CLI: content-defined +//! chunking, block packing, the reconstruction object, and the signed +//! release manifest. +//! +//! Files are split by content, not by size: chunk boundaries land where the +//! bytes say they land, so appending or editing part of a weight file leaves +//! every other chunk identical. Chunks are the dedup unit; blocks (sequences +//! of chunks sealed near a target size) are the storage unit. Both are named +//! by the sha256 of their bytes, computed by whoever holds them and +//! recomputed by the node on receipt. + +pub(crate) use crate::cid::sha256_hex; + +pub mod chunking; +pub mod manifest; +pub mod reconstruction; + +/// The schema tag carried by the reconstruction object defined here. Unknown +/// tags are rejected on parse so a reader never silently misreads a later +/// format. +pub const RECONSTRUCTION_SCHEMA_V1: &str = "gitlawb/model-reconstruction/v1"; + +/// Deterministic pseudo-random bytes for tests: `rand` is a dependency of +/// this crate, but a fixed-pattern generator keeps the tests reproducible. +#[cfg(test)] +pub(crate) fn gen_bytes(n: usize, seed: u64) -> Vec { + let mut v = Vec::with_capacity(n); + let mut x = seed; + while v.len() < n { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + v.extend_from_slice(&x.to_le_bytes()); + } + v.truncate(n); + v +} diff --git a/crates/twigpine-core/src/model/reconstruction.rs b/crates/twigpine-core/src/model/reconstruction.rs new file mode 100644 index 00000000..70d2a33e --- /dev/null +++ b/crates/twigpine-core/src/model/reconstruction.rs @@ -0,0 +1,788 @@ +//! The reconstruction object: versioned JSON mapping each chunk-stored file +//! to the block terms that rebuild it. +//! +//! A term names a block by its sha256 and a half-open range of that block's +//! chunk list; a file is its terms in order. Each block's own entry lists +//! every chunk's sha256 and length, so byte offsets inside a block are prefix +//! sums and the format carries no redundant offset column. +//! +//! The schema string is part of the signed surface: a reader rejects a tag it +//! does not know rather than guessing at a later layout. + +use std::collections::HashMap; +use std::io::{Read, Write}; + +use serde::{Deserialize, Serialize}; +use sha2::Digest; +use thiserror::Error; + +use super::chunking::{chunk_reader, Block, BLOCK_TARGET}; +use super::{sha256_hex, RECONSTRUCTION_SCHEMA_V1}; + +#[derive(Debug, Error)] +pub enum ModelError { + #[error("unsupported reconstruction schema {0:?}")] + UnknownSchema(String), + #[error("invalid reconstruction: {0}")] + Invalid(String), + #[error("hash mismatch: {0}")] + HashMismatch(String), + #[error("i/o error: {0}")] + Io(#[from] std::io::Error), + #[error("json error: {0}")] + Json(#[from] serde_json::Error), +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct Term { + /// sha256 of the block this term reads from. + pub block: String, + /// Half-open index range into that block's `chunks` list. + pub chunk_start: u32, + pub chunk_end: u32, + /// File bytes this term contributes: the sum of its chunks' lengths. + pub length: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct FileEntry { + pub path: String, + pub size: u64, + pub sha256: String, + pub terms: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ChunkMeta { + pub sha256: String, + pub len: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct BlockEntry { + pub sha256: String, + pub size: u64, + pub chunks: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct Reconstruction { + pub schema: String, + pub files: Vec, + pub blocks: Vec, +} + +impl Reconstruction { + pub fn new() -> Self { + Self { + schema: RECONSTRUCTION_SCHEMA_V1.to_string(), + files: Vec::new(), + blocks: Vec::new(), + } + } + + /// Parse and validate a stored object. Unknown schema tags are refused: + /// misreading a later layout silently is worse than failing. + pub fn parse(bytes: &[u8]) -> Result { + let this: Reconstruction = serde_json::from_slice(bytes)?; + if this.schema != RECONSTRUCTION_SCHEMA_V1 { + return Err(ModelError::UnknownSchema(this.schema)); + } + this.validate()?; + Ok(this) + } + + pub fn to_bytes(&self) -> Result, ModelError> { + Ok(serde_json::to_vec(self)?) + } + + /// Structural sanity: terms index real blocks, chunk ranges are ordered, + /// and each file's terms sum to its declared size. + pub fn validate(&self) -> Result<(), ModelError> { + let blocks: HashMap<&str, &BlockEntry> = + self.blocks.iter().map(|b| (b.sha256.as_str(), b)).collect(); + for file in &self.files { + let mut total = 0u64; + for term in &file.terms { + if term.chunk_start >= term.chunk_end { + return Err(ModelError::Invalid(format!( + "{}: empty chunk range in term for block {}", + file.path, term.block + ))); + } + let block = blocks.get(term.block.as_str()).ok_or_else(|| { + ModelError::Invalid(format!( + "{}: term names unknown block {}", + file.path, term.block + )) + })?; + if term.chunk_end as usize > block.chunks.len() { + return Err(ModelError::Invalid(format!( + "{}: chunk range {}..{} exceeds block {} with {} chunks", + file.path, + term.chunk_start, + term.chunk_end, + term.block, + block.chunks.len() + ))); + } + let span: u64 = block.chunks[term.chunk_start as usize..term.chunk_end as usize] + .iter() + .map(|c| c.len) + .sum(); + if span != term.length { + return Err(ModelError::Invalid(format!( + "{}: term length {span} != declared {}", + file.path, term.length + ))); + } + total += term.length; + } + if total != file.size { + return Err(ModelError::Invalid(format!( + "{}: terms cover {total} bytes, file declares {}", + file.path, file.size + ))); + } + } + Ok(()) + } + + fn block_index(&self) -> HashMap<&str, &BlockEntry> { + self.blocks.iter().map(|b| (b.sha256.as_str(), b)).collect() + } +} + +impl Default for Reconstruction { + fn default() -> Self { + Self::new() + } +} + +/// Chunk-sha lookup over prior reconstruction objects, the dedup surface: +/// a chunk whose hash a prior release already stored can be referenced by +/// term instead of re-uploaded. +#[derive(Default)] +pub struct DedupIndex { + /// chunk sha256 -> (block sha256, chunk index inside that block) + by_chunk: HashMap, +} + +impl DedupIndex { + pub fn new() -> Self { + Self::default() + } + + pub fn from_reconstructions(priors: &[&Reconstruction]) -> Self { + let mut idx = Self::new(); + for rec in priors { + idx.add(rec); + } + idx + } + + pub fn add(&mut self, rec: &Reconstruction) { + for block in &rec.blocks { + for (i, chunk) in block.chunks.iter().enumerate() { + // First writer wins: two priors carrying the same chunk are + // interchangeable, so keep the earliest. + self.by_chunk + .entry(chunk.sha256.clone()) + .or_insert((block.sha256.clone(), i as u32)); + } + } + } +} + +/// Seal the accumulating block of new chunks: patch its id into the terms +/// written while it was still open, then append the block-table entry and +/// queue the bytes for upload. +fn seal_pending( + pending: &mut Vec, + terms: &mut [Term], + block_entries: &mut Vec, + new_blocks: &mut Vec, +) { + if pending.is_empty() { + return; + } + let chunks = std::mem::take(pending); + let mut data = Vec::with_capacity(chunks.iter().map(|c| c.len).sum::()); + for c in &chunks { + data.extend_from_slice(&c.data); + } + let block = Block { + sha256: sha256_hex(&data), + chunks, + data, + }; + // Terms for new chunks were written with an empty placeholder id + // because the block hash is not known until it seals. + for term in terms.iter_mut() { + if term.block.is_empty() { + term.block = block.sha256.clone(); + } + } + block_entries.push(BlockEntry { + sha256: block.sha256.clone(), + size: block.data.len() as u64, + chunks: block + .chunks + .iter() + .map(|c| ChunkMeta { + sha256: c.sha256.clone(), + len: c.len as u64, + }) + .collect(), + }); + new_blocks.push(block); +} + +/// What `pack_file` produced: the file's entry, the new blocks to store and +/// upload, and the block entries the reconstruction object must carry. +pub struct PackedFile { + pub file: FileEntry, + pub new_blocks: Vec, + pub block_entries: Vec, +} + +/// Chunk and pack one file, reusing terms for chunks a prior reconstruction +/// already holds (`dedup`). `path` is the repo-relative path recorded in the +/// entry. +pub fn pack_file( + path: &str, + reader: R, + dedup: &DedupIndex, +) -> std::io::Result { + let mut file_hasher = sha2::Sha256::new(); + + let mut terms: Vec = Vec::new(); + let mut new_blocks: Vec = Vec::new(); + let mut block_entries: Vec = Vec::new(); + // Chunks not found in `dedup` accumulate into one pending block that + // seals only at the target size or EOF. Sealing on every reused-chunk + // boundary would scatter a fine-tune's edits into one tiny block each, + // multiplying the object and pin count blocks exist to keep low. + let mut pending: Vec = Vec::new(); + let mut pending_bytes: usize = 0; + let mut file_size: u64 = 0; + + for chunk in chunk_reader(reader) { + let chunk = chunk?; + file_hasher.update(&chunk.data); + file_size += chunk.len as u64; + if let Some((block_sha, idx)) = dedup.by_chunk.get(&chunk.sha256) { + // Extend the previous term when it already reads the chunk before + // this one in the same block: a shared run stays a single term. + match terms.last_mut() { + Some(t) if t.block == *block_sha && t.chunk_end == *idx => { + t.chunk_end = idx + 1; + t.length += chunk.len as u64; + } + _ => terms.push(Term { + block: block_sha.clone(), + chunk_start: *idx, + chunk_end: idx + 1, + length: chunk.len as u64, + }), + } + } else { + // The term indexes into the pending block at the position the + // chunk will hold once the block seals; the id is patched then. + let idx = pending.len() as u32; + let len = chunk.len as u64; + pending_bytes += chunk.len; + pending.push(chunk); + match terms.last_mut() { + Some(t) if t.block.is_empty() => { + t.chunk_end = idx + 1; + t.length += len; + } + _ => terms.push(Term { + block: String::new(), + chunk_start: idx, + chunk_end: idx + 1, + length: len, + }), + } + if pending_bytes >= BLOCK_TARGET { + seal_pending( + &mut pending, + &mut terms, + &mut block_entries, + &mut new_blocks, + ); + pending_bytes = 0; + } + } + } + seal_pending( + &mut pending, + &mut terms, + &mut block_entries, + &mut new_blocks, + ); + + let file = FileEntry { + path: path.to_string(), + size: file_size, + sha256: hex::encode(file_hasher.finalize()), + terms, + }; + Ok(PackedFile { + file, + new_blocks, + block_entries, + }) +} + +/// Byte-range mapping: which slices of which blocks serve file bytes +/// `[start, end_exclusive)`. Each slice is a contiguous run of whole or +/// clipped chunks inside one block. +#[derive(Debug, PartialEq, Eq)] +pub struct BlockSlice { + pub block: String, + /// Byte offset of the slice inside the block's chunk bytes. + pub offset: u64, + pub len: u64, +} + +/// Byte offset of each chunk inside a block, as a prefix sum over the +/// chunk table: `offsets[i]` is where chunk `i`'s bytes begin. +fn chunk_offsets(block: &BlockEntry) -> Vec { + block + .chunks + .iter() + .scan(0u64, |off, c| { + let start = *off; + *off += c.len; + Some(start) + }) + .collect() +} + +/// Byte range inside a block covering the chunks a term names. +fn term_span(block: &BlockEntry, term: &Term) -> Result<(u64, u64), ModelError> { + let start = term.chunk_start as usize; + let end = term.chunk_end as usize; + if start >= end || end > block.chunks.len() { + return Err(ModelError::Invalid(format!( + "term chunk range {start}..{end} outside block {} with {} chunks", + term.block, + block.chunks.len() + ))); + } + let offsets = chunk_offsets(block); + let len: u64 = block.chunks[start..end].iter().map(|c| c.len).sum(); + Ok((offsets[start], len)) +} + +pub fn map_file_range( + file: &FileEntry, + rec: &Reconstruction, + start: u64, + end_exclusive: u64, +) -> Result, ModelError> { + if start > end_exclusive || end_exclusive > file.size { + return Err(ModelError::Invalid(format!( + "range {start}..{end_exclusive} outside {} bytes of {}", + file.size, file.path + ))); + } + let blocks = rec.block_index(); + let mut out: Vec = Vec::new(); + let mut cursor = 0u64; + for term in &file.terms { + let block = blocks + .get(term.block.as_str()) + .ok_or_else(|| ModelError::Invalid(format!("unknown block {}", term.block)))?; + let term_end = cursor + term.length; + if term_end <= start || cursor >= end_exclusive { + cursor = term_end; + continue; + } + let offsets = chunk_offsets(block); + for (i, chunk) in block.chunks[term.chunk_start as usize..term.chunk_end as usize] + .iter() + .enumerate() + { + // This chunk covers file bytes [chunk_lo, chunk_hi). + let chunk_lo = cursor; + let chunk_hi = cursor + chunk.len; + cursor = chunk_hi; + let s = start.max(chunk_lo); + let e = end_exclusive.min(chunk_hi); + if s < e { + let off = offsets[term.chunk_start as usize + i] + (s - chunk_lo); + match out.last_mut() { + // Adjacent slices in the same block merge into one read. + Some(slice) if slice.block == term.block && slice.offset + slice.len == off => { + let slice: &mut BlockSlice = slice; + slice.len += e - s; + } + _ => out.push(BlockSlice { + block: term.block.clone(), + offset: off, + len: e - s, + }), + } + } + } + cursor = term_end; + } + Ok(out) +} + +/// A range read over a stored block: `fetch(block_id, offset, len)` returns +/// exactly `len` bytes starting at `offset`. +pub type BlockRangeFetch<'a> = dyn FnMut(&str, u64, u64) -> std::io::Result> + 'a; + +/// Rebuild a file's bytes from its terms, verifying every chunk's hash and +/// the whole-file digest. `fetch` is a range read returning exactly `len` +/// bytes starting at `offset` inside the named block (a BlockStore range +/// read or a cache hit on the caller's side). The block id itself is not +/// re-verified here: blocks are hash-verified when stored, and the chunk +/// hashes cover the bytes this file actually uses. +pub fn rebuild_file( + file: &FileEntry, + rec: &Reconstruction, + fetch: &mut BlockRangeFetch<'_>, + out: &mut W, +) -> Result { + let blocks = rec.block_index(); + let mut file_hasher = sha2::Sha256::new(); + let mut written = 0u64; + for term in &file.terms { + let block = blocks + .get(term.block.as_str()) + .ok_or_else(|| ModelError::Invalid(format!("unknown block {}", term.block)))?; + let (offset, span_len) = term_span(block, term)?; + let bytes = fetch(&term.block, offset, span_len)?; + if bytes.len() as u64 != span_len { + return Err(ModelError::Invalid(format!( + "range read on block {} returned {} of {span_len} bytes", + term.block, + bytes.len() + ))); + } + let mut rel = 0usize; + for chunk in &block.chunks[term.chunk_start as usize..term.chunk_end as usize] { + let data = &bytes[rel..rel + chunk.len as usize]; + if sha256_hex(data) != chunk.sha256 { + return Err(ModelError::HashMismatch(format!( + "chunk {} in block {}", + chunk.sha256, term.block + ))); + } + out.write_all(data)?; + file_hasher.update(data); + written += chunk.len; + rel += chunk.len as usize; + } + } + if written != file.size { + return Err(ModelError::Invalid(format!( + "{}: rebuilt {written} bytes, declared {}", + file.path, file.size + ))); + } + let digest = hex::encode(file_hasher.finalize()); + if digest != file.sha256 { + return Err(ModelError::HashMismatch(format!( + "{}: rebuilt bytes do not hash to the manifest id", + file.path + ))); + } + Ok(written) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::model::gen_bytes; + use std::io::Cursor; + + fn pack(path: &str, bytes: &[u8]) -> (Reconstruction, PackedFile) { + let dedup = DedupIndex::new(); + let packed = pack_file(path, Cursor::new(bytes.to_vec()), &dedup).unwrap(); + let mut rec = Reconstruction::new(); + rec.files.push(packed.file.clone()); + rec.blocks.extend(packed.block_entries.clone()); + (rec, packed) + } + + #[test] + fn pack_then_rebuild_is_byte_identical() { + let bytes = gen_bytes(3 * 1024 * 1024 + 13, 7); + let (rec, packed) = pack("model.safetensors", &bytes); + rec.validate().unwrap(); + let store: HashMap> = packed + .new_blocks + .iter() + .map(|b| (b.sha256.clone(), b.data.clone())) + .collect(); + let mut out = Vec::new(); + let n = rebuild_file( + &packed.file, + &rec, + &mut |id, off, len| { + let block = store.get(id).expect("block"); + Ok(block[off as usize..(off + len) as usize].to_vec()) + }, + &mut out, + ) + .unwrap(); + assert_eq!(n as usize, bytes.len()); + assert_eq!(out, bytes); + } + + #[test] + fn reconstruction_round_trips_json_and_rejects_unknown_schema() { + let bytes = gen_bytes(600_000, 11); + let (rec, _) = pack("weights.bin", &bytes); + let parsed = Reconstruction::parse(&rec.to_bytes().unwrap()).unwrap(); + assert_eq!(parsed, rec); + + let mut bad = rec.clone(); + bad.schema = "gitlawb/model-reconstruction/v99".to_string(); + assert!(matches!( + Reconstruction::parse(&bad.to_bytes().unwrap()), + Err(ModelError::UnknownSchema(_)) + )); + } + + #[test] + fn dedup_reuses_prior_chunks_without_new_blocks() { + // A fine-tune sharing most of the base's bytes: only the changed + // chunks seal into new blocks, the rest become terms into the prior + // reconstruction's blocks. + let base = gen_bytes(4 * 1024 * 1024, 23); + let (base_rec, base_packed) = pack("base.safetensors", &base); + assert_eq!(base_packed.new_blocks.len(), 1); + + let mut tuned = base.clone(); + // Overwrite a 300 KiB region in the middle: FastCDC boundaries + // resync within a few chunks. + for b in tuned.iter_mut().skip(2 * 1024 * 1024).take(300 * 1024) { + *b = b.wrapping_add(1); + } + + let dedup = DedupIndex::from_reconstructions(&[&base_rec]); + let packed = pack_file("tuned.safetensors", Cursor::new(tuned.clone()), &dedup).unwrap(); + let mut rec = Reconstruction::new(); + // The block table carries prior blocks too: terms point at them, so + // the object must list them for offsets to resolve. + rec.blocks.extend(base_rec.blocks.iter().cloned()); + rec.blocks.extend(packed.block_entries.clone()); + rec.files.push(packed.file.clone()); + rec.validate().unwrap(); + + let reused_chunks: usize = packed + .file + .terms + .iter() + .filter(|t| t.block == base_rec.blocks[0].sha256) + .map(|t| (t.chunk_end - t.chunk_start) as usize) + .sum(); + let total_chunks: usize = packed + .file + .terms + .iter() + .map(|t| (t.chunk_end - t.chunk_start) as usize) + .sum(); + assert!( + reused_chunks * 10 >= total_chunks * 8, + "expected most chunks reused, got {reused_chunks}/{total_chunks}" + ); + assert!( + packed.new_blocks.len() <= 1, + "changed chunks should seal into at most one new block" + ); + + let store: HashMap> = base_packed + .new_blocks + .iter() + .chain(packed.new_blocks.iter()) + .map(|b| (b.sha256.clone(), b.data.clone())) + .collect(); + let mut out = Vec::new(); + rebuild_file( + &packed.file, + &rec, + &mut |id, off, len| { + let block = store.get(id).expect("block"); + Ok(block[off as usize..(off + len) as usize].to_vec()) + }, + &mut out, + ) + .unwrap(); + assert_eq!(out, tuned); + } + + #[test] + fn scattered_edits_share_one_new_block() { + // A fine-tune whose edits are spread across the file: every run of + // changed chunks lands in the same pending block, so the object and + // pin count does not grow with the number of edited regions. + let base = gen_bytes(20 * 1024 * 1024, 41); + let (base_rec, base_packed) = pack("model.safetensors", &base); + let region = (base.len() - 200_000) / 50; + let mut tuned = base.clone(); + for i in 0..50 { + let at = 50_000 + i * region; + for b in tuned.iter_mut().skip(at).take(4_096) { + *b = b.wrapping_add(7); + } + } + + let dedup = DedupIndex::from_reconstructions(&[&base_rec]); + let packed = pack_file("tuned.safetensors", Cursor::new(tuned.clone()), &dedup).unwrap(); + assert_eq!( + packed.new_blocks.len(), + 1, + "scattered edits must seal into one new block, not one per run" + ); + + let mut rec = Reconstruction::new(); + rec.blocks.extend(base_rec.blocks.iter().cloned()); + rec.blocks.extend(packed.block_entries.clone()); + rec.files.push(packed.file.clone()); + rec.validate().unwrap(); + + let store: HashMap> = base_packed + .new_blocks + .iter() + .chain(packed.new_blocks.iter()) + .map(|b| (b.sha256.clone(), b.data.clone())) + .collect(); + let mut out = Vec::new(); + rebuild_file( + &packed.file, + &rec, + &mut |id, off, len| { + let block = store.get(id).expect("block"); + Ok(block[off as usize..(off + len) as usize].to_vec()) + }, + &mut out, + ) + .unwrap(); + assert_eq!(out, tuned); + } + + #[test] + fn range_mapping_hits_the_middle_of_a_block() { + // Build a 10-chunk block synthetically so the middle-subrange case is + // exact, not dependent on where FastCDC happens to cut. + let chunk_bytes: Vec> = (0..10) + .map(|i| gen_bytes(100 + i * 10, i as u64 + 1)) + .collect(); + let chunks: Vec = chunk_bytes + .iter() + .map(|c| ChunkMeta { + sha256: sha256_hex(c), + len: c.len() as u64, + }) + .collect(); + let block_data: Vec = chunk_bytes.concat(); + let block_sha = sha256_hex(&block_data); + // A file whose whole content is one term referencing chunks 3..6 of + // a 10-chunk block: the shape dedup produces when a prior release + // already stored the surrounding chunks. + let term_len: u64 = chunks[3..6].iter().map(|c| c.len).sum(); + let prefix: u64 = chunks[..3].iter().map(|c| c.len).sum(); + let file_bytes: Vec = chunk_bytes[3..6].concat(); + let file = FileEntry { + path: "f".into(), + size: term_len, + sha256: sha256_hex(&file_bytes), + terms: vec![Term { + block: block_sha.clone(), + chunk_start: 3, + chunk_end: 6, + length: term_len, + }], + }; + let rec = Reconstruction { + schema: RECONSTRUCTION_SCHEMA_V1.into(), + files: vec![file.clone()], + blocks: vec![BlockEntry { + sha256: block_sha.clone(), + size: block_data.len() as u64, + chunks, + }], + }; + rec.validate().unwrap(); + + // A middle range of the file maps to the middle of the block: the + // slice offset includes the prefix sum of chunks 0..3 that the term + // skips. + let want = file.size - 10; + let slices = map_file_range(&file, &rec, 10, want).unwrap(); + let served: u64 = slices.iter().map(|s| s.len).sum(); + assert_eq!(served, want - 10); + assert_eq!(slices[0].offset, prefix + 10); + let mut got = Vec::new(); + for s in &slices { + got.extend_from_slice(&block_data[s.offset as usize..(s.offset + s.len) as usize]); + } + assert_eq!(got, file_bytes[10..want as usize]); + } + + #[test] + fn rebuild_rejects_a_corrupt_block() { + let bytes = gen_bytes(300_000, 31); + let (rec, packed) = pack("f", &bytes); + let mut out = Vec::new(); + let err = rebuild_file( + &packed.file, + &rec, + &mut |_id, _off, len| Ok(vec![0u8; len as usize]), + &mut out, + ) + .unwrap_err(); + assert!(matches!(err, ModelError::HashMismatch(_))); + } + + #[test] + fn rebuild_fetches_only_the_bytes_the_file_needs() { + // Terms can revisit one block or name a middle chunk range: the + // fetcher must serve each term's span, never whole blocks, so the + // bytes read equal the file size rather than a multiple of it. + let base = gen_bytes(4 * 1024 * 1024, 53); + let (base_rec, base_packed) = pack("base.safetensors", &base); + let mut tuned = base.clone(); + for b in tuned.iter_mut().skip(1024 * 1024).take(64 * 1024) { + *b = b.wrapping_add(3); + } + + let dedup = DedupIndex::from_reconstructions(&[&base_rec]); + let packed = pack_file("tuned.safetensors", Cursor::new(tuned.clone()), &dedup).unwrap(); + let mut rec = Reconstruction::new(); + rec.blocks.extend(base_rec.blocks.iter().cloned()); + rec.blocks.extend(packed.block_entries.clone()); + rec.files.push(packed.file.clone()); + rec.validate().unwrap(); + + let store: HashMap> = base_packed + .new_blocks + .iter() + .chain(packed.new_blocks.iter()) + .map(|b| (b.sha256.clone(), b.data.clone())) + .collect(); + let mut fetched = 0u64; + let mut out = Vec::new(); + rebuild_file( + &packed.file, + &rec, + &mut |id, off, len| { + fetched += len; + let block = store.get(id).expect("block"); + Ok(block[off as usize..(off + len) as usize].to_vec()) + }, + &mut out, + ) + .unwrap(); + assert_eq!(out, tuned); + assert_eq!(fetched, packed.file.size); + } +} diff --git a/crates/twigpine-core/tests/fixtures/model-release-v1/manifest.json b/crates/twigpine-core/tests/fixtures/model-release-v1/manifest.json new file mode 100644 index 00000000..0a2d2126 --- /dev/null +++ b/crates/twigpine-core/tests/fixtures/model-release-v1/manifest.json @@ -0,0 +1 @@ +{"schema":"gitlawb/model-release/v1","repo":"alice/llm","publisher_did":"did:key:z6MkgAnvkP45uNxwCKeNdt6wrYkEjpYX4f7Nrd8MQqFL8Fbn","license":"apache-2.0","parent":null,"files":[{"path":"model.safetensors","size":1024,"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","format":"safetensors","storage":"chunked"},{"path":"config.json","size":256,"sha256":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","format":"text","storage":"git"}],"reconstruction":{"sha256":"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc","size":512}} \ No newline at end of file diff --git a/crates/twigpine-core/tests/fixtures/model-release-v1/manifest.sig b/crates/twigpine-core/tests/fixtures/model-release-v1/manifest.sig new file mode 100644 index 00000000..b30b30e1 --- /dev/null +++ b/crates/twigpine-core/tests/fixtures/model-release-v1/manifest.sig @@ -0,0 +1 @@ +dri5eq548Bth3n7iqOI6Z2fsD5TLmD5bK1Ru/svQNhMnFz0EGaexMIe0uZ+HMLgmVWC6dSccbpfXlbNmS7r1Cw== \ No newline at end of file diff --git a/crates/twigpine-core/tests/fixtures/model-release-v1/publisher.did b/crates/twigpine-core/tests/fixtures/model-release-v1/publisher.did new file mode 100644 index 00000000..db73a6f5 --- /dev/null +++ b/crates/twigpine-core/tests/fixtures/model-release-v1/publisher.did @@ -0,0 +1 @@ +did:key:z6MkgAnvkP45uNxwCKeNdt6wrYkEjpYX4f7Nrd8MQqFL8Fbn \ No newline at end of file diff --git a/crates/twigpine-node/src/auth/mod.rs b/crates/twigpine-node/src/auth/mod.rs index eae95884..24c09296 100644 --- a/crates/twigpine-node/src/auth/mod.rs +++ b/crates/twigpine-node/src/auth/mod.rs @@ -1303,6 +1303,7 @@ mod tests { crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), task_read_rate_limiter: RateLimiter::new(1200, Duration::from_secs(3600)), git_bin: "git".to_string(), + block_store: None, } } diff --git a/crates/twigpine-node/src/blockstore/fs.rs b/crates/twigpine-node/src/blockstore/fs.rs new file mode 100644 index 00000000..61da56b1 --- /dev/null +++ b/crates/twigpine-node/src/blockstore/fs.rs @@ -0,0 +1,369 @@ +//! Filesystem block backend: objects under `///` with +//! staged writes in `/tmp/`. +//! +//! A write lands in a uniquely-named temp file, is fsynced, and becomes +//! visible only through a hard link to its content address, which is atomic +//! and refuses to overwrite: two publishers racing the same object get one +//! `Stored` and one `AlreadyPresent`, never a torn object. The temp file is +//! removed on every failure path, so a rejected body leaves nothing behind. + +use std::io; +use std::path::{Path, PathBuf}; + +use async_trait::async_trait; +use bytes::Bytes; +use futures::stream::BoxStream; +use futures::StreamExt; +use sha2::{Digest, Sha256}; +use tokio::io::AsyncWriteExt; + +use super::{BlockId, BlockStore, BlockStoreError, PutOutcome}; + +pub struct FsBlockStore { + root: PathBuf, +} + +impl FsBlockStore { + pub fn new(root: PathBuf) -> io::Result { + std::fs::create_dir_all(root.join("tmp"))?; + Ok(Self { root }) + } + + fn content_path(&self, id: &BlockId) -> PathBuf { + let s = id.as_str(); + self.root.join(&s[..2]).join(&s[2..4]).join(s) + } + + fn temp_path(&self) -> PathBuf { + self.root + .join("tmp") + .join(format!("put-{}", uuid::Uuid::new_v4())) + } + + async fn cleanup(path: &Path) { + let _ = tokio::fs::remove_file(path).await; + } +} + +/// Fsync a directory so a name created inside it survives a crash: after +/// `Stored`, a link that only exists in the page cache is a lost write. +#[cfg(unix)] +async fn fsync_dir(dir: &Path) -> io::Result<()> { + tokio::fs::File::open(dir).await?.sync_all().await +} + +/// Directory fsync is a unix durability primitive; elsewhere the link is +/// the best guarantee available. +#[cfg(not(unix))] +async fn fsync_dir(_: &Path) -> io::Result<()> { + Ok(()) +} + +#[async_trait] +impl BlockStore for FsBlockStore { + async fn put_verified( + &self, + id: &BlockId, + mut body: BoxStream<'static, io::Result>, + max_len: u64, + ) -> Result { + let dest = self.content_path(id); + if dest.exists() { + return Ok(PutOutcome::AlreadyPresent); + } + + let tmp = self.temp_path(); + let mut file = match tokio::fs::File::create(&tmp).await { + Ok(f) => f, + Err(e) => return Err(e.into()), + }; + let mut hasher = Sha256::new(); + let mut len: u64 = 0; + let result: Result<(), BlockStoreError> = async { + while let Some(chunk) = body.next().await { + let chunk = chunk?; + len = len.saturating_add(chunk.len() as u64); + if len > max_len { + return Err(BlockStoreError::TooLarge(max_len)); + } + hasher.update(&chunk); + file.write_all(&chunk).await?; + } + Ok(()) + } + .await; + + if let Err(e) = result { + drop(file); + Self::cleanup(&tmp).await; + return Err(e); + } + if hex::encode(hasher.finalize()) != id.as_str() { + drop(file); + Self::cleanup(&tmp).await; + return Err(BlockStoreError::DigestMismatch); + } + // Flush before the link: a crash between link and write-back would + // leave a truncated object under a valid id. + if let Err(e) = file.sync_all().await { + drop(file); + Self::cleanup(&tmp).await; + return Err(e.into()); + } + drop(file); + + let dir = dest.parent().expect("content path has two levels"); + let fresh_dirs = !dir.exists(); + if let Err(e) = tokio::fs::create_dir_all(dir).await { + Self::cleanup(&tmp).await; + return Err(e.into()); + } + if fresh_dirs { + // The new / levels are themselves names in a parent + // directory; persist each level's entry. + for ancestor in [dir.parent().unwrap_or(&self.root), self.root.as_path()] { + if let Err(e) = fsync_dir(ancestor).await { + Self::cleanup(&tmp).await; + return Err(e.into()); + } + } + } + // A hard link refuses to replace an existing object, which is what + // makes a same-content race benign rather than a torn write. + match tokio::fs::hard_link(&tmp, &dest).await { + Ok(()) => { + if let Err(e) = fsync_dir(dir).await { + Self::cleanup(&tmp).await; + return Err(e.into()); + } + Self::cleanup(&tmp).await; + Ok(PutOutcome::Stored) + } + Err(e) if e.kind() == io::ErrorKind::AlreadyExists => { + Self::cleanup(&tmp).await; + Ok(PutOutcome::AlreadyPresent) + } + Err(e) => { + Self::cleanup(&tmp).await; + Err(e.into()) + } + } + } + + async fn get_range( + &self, + id: &BlockId, + start: u64, + end_exclusive: u64, + ) -> Result>, BlockStoreError> { + use tokio::io::{AsyncReadExt, AsyncSeekExt}; + let path = self.content_path(id); + let mut file = match tokio::fs::File::open(&path).await { + Ok(f) => f, + Err(e) if e.kind() == io::ErrorKind::NotFound => return Err(BlockStoreError::NotFound), + Err(e) => return Err(e.into()), + }; + let len = file.metadata().await?.len(); + if start > end_exclusive || end_exclusive > len { + return Err(BlockStoreError::OutOfRange); + } + file.seek(io::SeekFrom::Start(start)).await?; + let limited = file.take(end_exclusive - start); + Ok(tokio_util::io::ReaderStream::new(limited).boxed()) + } + + async fn size(&self, id: &BlockId) -> Result, BlockStoreError> { + match tokio::fs::metadata(self.content_path(id)).await { + Ok(m) => Ok(Some(m.len())), + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(None), + Err(e) => Err(e.into()), + } + } + + async fn delete(&self, id: &BlockId) -> Result<(), BlockStoreError> { + match tokio::fs::remove_file(self.content_path(id)).await { + Ok(()) => Ok(()), + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(e.into()), + } + } + + fn backend_name(&self) -> &'static str { + "fs" + } +} + +#[cfg(test)] +mod tests { + use super::*; + use sha2::Digest; + + fn id_of(bytes: &[u8]) -> BlockId { + BlockId::parse(&hex::encode(Sha256::digest(bytes))).unwrap() + } + + fn stream(bytes: Vec) -> BoxStream<'static, io::Result> { + futures::stream::iter(vec![Ok(Bytes::from(bytes))]).boxed() + } + + fn chunked_stream(parts: Vec>) -> BoxStream<'static, io::Result> { + futures::stream::iter(parts.into_iter().map(|p| Ok(Bytes::from(p)))).boxed() + } + + async fn collect(mut s: BoxStream<'static, io::Result>) -> Vec { + let mut out = Vec::new(); + while let Some(c) = s.next().await { + out.extend_from_slice(&c.unwrap()); + } + out + } + + fn store(dir: &tempfile::TempDir) -> FsBlockStore { + FsBlockStore::new(dir.path().join("blocks")).unwrap() + } + + #[tokio::test] + async fn put_get_size_roundtrip() { + let dir = tempfile::tempdir().unwrap(); + let s = store(&dir); + let body: Vec = (0..100_000u32).flat_map(|i| i.to_le_bytes()).collect(); + let id = id_of(&body); + assert_eq!( + s.put_verified(&id, stream(body.clone()), u64::MAX) + .await + .unwrap(), + PutOutcome::Stored + ); + assert_eq!(s.size(&id).await.unwrap(), Some(body.len() as u64)); + assert_eq!(s.backend_name(), "fs"); + // A second put of the same content is a no-op. + assert_eq!( + s.put_verified(&id, stream(body.clone()), u64::MAX) + .await + .unwrap(), + PutOutcome::AlreadyPresent + ); + } + + #[tokio::test] + async fn wrong_id_is_rejected_and_leaves_nothing() { + let dir = tempfile::tempdir().unwrap(); + let s = store(&dir); + let id = id_of(b"different bytes"); + let err = s + .put_verified(&id, stream(b"actual".to_vec()), u64::MAX) + .await + .unwrap_err(); + assert!(matches!(err, BlockStoreError::DigestMismatch)); + assert_eq!(s.size(&id).await.unwrap(), None); + // No temp file survives the failure. + assert_eq!( + std::fs::read_dir(dir.path().join("blocks/tmp")) + .unwrap() + .count(), + 0 + ); + } + + #[tokio::test] + async fn oversized_body_is_rejected_while_streaming() { + let dir = tempfile::tempdir().unwrap(); + let s = store(&dir); + let body = vec![0u8; 1024]; + let id = id_of(&body); + let err = s.put_verified(&id, stream(body), 512).await.unwrap_err(); + assert!(matches!(err, BlockStoreError::TooLarge(512))); + assert_eq!(s.size(&id).await.unwrap(), None); + } + + #[tokio::test] + async fn range_reads_return_exact_bytes() { + let dir = tempfile::tempdir().unwrap(); + let s = store(&dir); + let body: Vec = (0..255u8).collect(); + let id = id_of(&body); + s.put_verified(&id, stream(body.clone()), u64::MAX) + .await + .unwrap(); + + let got = collect(s.get_range(&id, 10, 20).await.unwrap()).await; + assert_eq!(got, body[10..20]); + + let got = collect(s.get_range(&id, 0, 255).await.unwrap()).await; + assert_eq!(got, body); + + assert!(matches!( + s.get_range(&id, 200, 300).await.map(|_| ()).unwrap_err(), + BlockStoreError::OutOfRange + )); + let missing = id_of(b"missing"); + assert!(matches!( + s.get_range(&missing, 0, 1).await.map(|_| ()).unwrap_err(), + BlockStoreError::NotFound + )); + } + + #[tokio::test] + async fn range_edge_contract_matches_the_trait() { + let dir = tempfile::tempdir().unwrap(); + let s = store(&dir); + crate::blockstore::parity::get_range_edge_contract(&s).await; + } + + #[tokio::test] + async fn concurrent_puts_leave_one_object() { + let dir = tempfile::tempdir().unwrap(); + let s = std::sync::Arc::new(store(&dir)); + let body: Vec = (0..50_000u32).map(|i| (i % 251) as u8).collect(); + let id = id_of(&body); + let (a, b) = tokio::join!( + s.put_verified(&id, stream(body.clone()), u64::MAX), + s.put_verified(&id, stream(body.clone()), u64::MAX) + ); + assert!(a.is_ok() && b.is_ok()); + assert_eq!(s.size(&id).await.unwrap(), Some(body.len() as u64)); + let got = collect(s.get_range(&id, 0, body.len() as u64).await.unwrap()).await; + assert_eq!(got, body); + } + + #[tokio::test] + async fn delete_removes_and_is_idempotent() { + let dir = tempfile::tempdir().unwrap(); + let s = store(&dir); + let id = id_of(b"gone"); + s.put_verified(&id, stream(b"gone".to_vec()), u64::MAX) + .await + .unwrap(); + s.delete(&id).await.unwrap(); + assert_eq!(s.size(&id).await.unwrap(), None); + s.delete(&id).await.unwrap(); + } + + #[tokio::test] + async fn streamed_chunks_hash_the_whole_body() { + let dir = tempfile::tempdir().unwrap(); + let s = store(&dir); + let body: Vec = (0..10_000u32).map(|i| (i % 256) as u8).collect(); + let parts: Vec> = body.chunks(777).map(|c| c.to_vec()).collect(); + let id = id_of(&body); + s.put_verified(&id, chunked_stream(parts), u64::MAX) + .await + .unwrap(); + let got = collect(s.get_range(&id, 0, body.len() as u64).await.unwrap()).await; + assert_eq!(got, body); + } + + #[test] + fn block_id_shape() { + assert!(BlockId::parse(&"a".repeat(64)).is_ok()); + for bad in [ + "abc", + &"A".repeat(64), + &"g".repeat(64), + &"a".repeat(63), + &"a".repeat(65), + ] { + assert!(BlockId::parse(bad).is_err(), "{bad} must be rejected"); + } + } +} diff --git a/crates/twigpine-node/src/blockstore/mod.rs b/crates/twigpine-node/src/blockstore/mod.rs new file mode 100644 index 00000000..4a89325d --- /dev/null +++ b/crates/twigpine-node/src/blockstore/mod.rs @@ -0,0 +1,233 @@ +//! Content-addressed block storage for model weights. +//! +//! A block is a sequence of content chunks packed into one object, named by +//! the sha256 of its bytes. The store is content-neutral: later content types +//! (datasets, assets) reuse it without a parallel access surface. +//! +//! Writes are verified, not trusted: a caller supplies the expected id and the +//! backend hashes the body as it streams, so a corrupted or malicious payload +//! is rejected and never becomes visible under its claimed name. Reads are +//! ranged, so a downloader resumes without re-serving whole blocks. + +use std::fmt; +use std::io; +use std::str::FromStr; +use std::sync::Arc; + +use async_trait::async_trait; +use bytes::Bytes; +use futures::stream::BoxStream; +use thiserror::Error; + +pub mod fs; +pub mod s3; + +/// A content address: exactly 64 lowercase hex characters, the sha256 of the +/// object's bytes. The shape is validated once at the boundary so every +/// backend can treat `id.as_str()` as a safe storage key fragment. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct BlockId(String); + +impl BlockId { + pub fn parse(s: &str) -> Result { + let valid = s.len() == 64 + && s.bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)); + if valid { + Ok(Self(s.to_string())) + } else { + Err(BlockIdError) + } + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl fmt::Display for BlockId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +impl FromStr for BlockId { + type Err = BlockIdError; + fn from_str(s: &str) -> Result { + Self::parse(s) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)] +#[error("a block id is exactly 64 lowercase hex characters")] +pub struct BlockIdError; + +/// What a verified write did. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PutOutcome { + /// The object's bytes were verified and are now stored under its id. + Stored, + /// An object already exists under this id. Because the id is the content + /// hash, the stored bytes are the same bytes, so nothing was written. + AlreadyPresent, +} + +#[derive(Debug, Error)] +pub enum BlockStoreError { + /// The streamed body did not hash to the id it was written under. Nothing + /// was stored. + #[error("body does not hash to the expected block id")] + DigestMismatch, + /// The body exceeded the caller-supplied cap while streaming. Nothing was + /// stored. + #[error("body exceeds the {0}-byte limit")] + TooLarge(u64), + /// The requested byte range is outside the stored object. + #[error("requested range is outside the object")] + OutOfRange, + /// No object exists under this id. + #[error("no object under this block id")] + NotFound, + /// A remote write's outcome cannot be determined: the request may have + /// committed while its answer was lost. Callers must treat the object as + /// possibly present and resolve with `size()` before retrying, exactly the + /// vocabulary `git::publish` uses for an upload whose commit status is + /// unknown. + #[error("the write's outcome is unknown; check size() before retrying")] + Unknown, + #[error("i/o error: {0}")] + Io(#[from] io::Error), + #[error("backend error: {0}")] + Backend(String), +} + +#[async_trait] +pub trait BlockStore: Send + Sync { + /// Stream `body` into the store under `id`, hashing as it goes. The body + /// is never fully buffered. Fails with `TooLarge` the moment the body + /// exceeds `max_len`, with `DigestMismatch` when the bytes do not hash to + /// `id`, and leaves no object or temporary state behind on any failure. + async fn put_verified( + &self, + id: &BlockId, + body: BoxStream<'static, io::Result>, + max_len: u64, + ) -> Result; + + /// Stream bytes `[start, end_exclusive)` of the stored object. An empty + /// range (`start == end_exclusive`) succeeds with an empty stream; + /// `start > end_exclusive` or an `end_exclusive` past the object size + /// fails with `OutOfRange`; a missing object fails with `NotFound`. + /// Backends must honor this contract identically. + async fn get_range( + &self, + id: &BlockId, + start: u64, + end_exclusive: u64, + ) -> Result>, BlockStoreError>; + + /// The stored object's length in bytes, or `None` when absent. + async fn size(&self, id: &BlockId) -> Result, BlockStoreError>; + + /// Remove the object. Deleting an absent object is not an error. + async fn delete(&self, id: &BlockId) -> Result<(), BlockStoreError>; + + /// Which backend this is ("fs" or "s3"), for logging and diagnostics. + fn backend_name(&self) -> &'static str; +} + +/// Build the configured backend, or `None` when model hosting is off. +/// +/// Fails closed: with the flag on, a backend that cannot be constructed is an +/// error the caller propagates, not a silent fall back to local disk (the +/// same posture purge-spam takes with the object store). +pub async fn build(config: &crate::config::Config) -> anyhow::Result>> { + if !config.model_hosting_enabled { + return Ok(None); + } + if !config.blockstore_s3_bucket.is_empty() { + let store = s3::S3BlockStore::new(&config.blockstore_s3_bucket).await?; + return Ok(Some(Arc::new(store))); + } + let store = fs::FsBlockStore::new(config.blockstore_dir.clone())?; + Ok(Some(Arc::new(store))) +} + +/// Assertions both backends must satisfy on `get_range` edges, run from each +/// backend's own test module against a live store so the trait contract +/// cannot drift between them. +#[cfg(test)] +pub(crate) mod parity { + use super::*; + use futures::StreamExt; + use sha2::{Digest, Sha256}; + + pub(crate) async fn get_range_edge_contract(store: &dyn BlockStore) { + let body: Vec = (0..10_000u32).map(|i| (i % 251) as u8).collect(); + let id = BlockId::parse(&hex::encode(Sha256::digest(&body))).unwrap(); + let put_body = futures::stream::iter(vec![Ok(Bytes::from(body.clone()))]).boxed(); + store.put_verified(&id, put_body, u64::MAX).await.unwrap(); + let size = body.len() as u64; + + async fn collect(mut s: BoxStream<'static, io::Result>) -> Vec { + let mut out = Vec::new(); + while let Some(c) = s.next().await { + out.extend_from_slice(&c.unwrap()); + } + out + } + + // Empty ranges are in bounds: at offset 0 and at the end. + assert!(collect(store.get_range(&id, 0, 0).await.unwrap()) + .await + .is_empty()); + assert!(collect(store.get_range(&id, size, size).await.unwrap()) + .await + .is_empty()); + // The whole object is the largest legal range. + assert_eq!( + collect(store.get_range(&id, 0, size).await.unwrap()).await, + body + ); + // One byte past the end is out of bounds, from any start. + assert!(matches!( + store + .get_range(&id, 0, size + 1) + .await + .map(|_| ()) + .unwrap_err(), + BlockStoreError::OutOfRange + )); + assert!(matches!( + store + .get_range(&id, 10, size + 1) + .await + .map(|_| ()) + .unwrap_err(), + BlockStoreError::OutOfRange + )); + // A reversed range is out of bounds. + assert!(matches!( + store.get_range(&id, 20, 10).await.map(|_| ()).unwrap_err(), + BlockStoreError::OutOfRange + )); + // A missing object is NotFound, even for an empty range. + let missing = BlockId::parse(&"0".repeat(64)).unwrap(); + assert!(matches!( + store + .get_range(&missing, 0, 1) + .await + .map(|_| ()) + .unwrap_err(), + BlockStoreError::NotFound + )); + assert!(matches!( + store + .get_range(&missing, 0, 0) + .await + .map(|_| ()) + .unwrap_err(), + BlockStoreError::NotFound + )); + } +} diff --git a/crates/twigpine-node/src/blockstore/s3.rs b/crates/twigpine-node/src/blockstore/s3.rs new file mode 100644 index 00000000..afafdbc2 --- /dev/null +++ b/crates/twigpine-node/src/blockstore/s3.rs @@ -0,0 +1,665 @@ +//! S3-compatible block backend: objects under `model-blocks/v1//`, +//! written through multipart upload. +//! +//! The verified-write contract maps onto multipart cleanly: parts go up while +//! the body streams (never fully buffered), and the object becomes visible +//! only when `CompleteMultipartUpload` commits, which runs after the hash +//! check. A mismatch aborts the upload instead, so a bad body never leaves an +//! object under a valid-looking id. +//! +//! The one case multipart cannot answer is a lost response: a timeout after +//! the complete request was sent may still have committed the object. That is +//! reported as [`BlockStoreError::Unknown`], the same "maybe sent" vocabulary +//! `git::publish` uses, and the caller resolves it with a HEAD rather than +//! assuming the write failed. + +use std::io; + +use async_trait::async_trait; +use aws_sdk_s3::error::SdkError; +use aws_sdk_s3::primitives::ByteStream; +use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart}; +use aws_sdk_s3::Client as S3Client; +use bytes::Bytes; +use futures::stream::BoxStream; +use futures::StreamExt; +use sha2::{Digest, Sha256}; +#[cfg(test)] +use std::time::Duration; + +use super::{BlockId, BlockStore, BlockStoreError, PutOutcome}; + +/// Multipart part size. Larger than the 5 MiB floor so a 128 MiB block is a +/// handful of parts, small enough that one part's retry cost stays bounded. +const PART_SIZE: usize = 16 * 1024 * 1024; + +const KEY_PREFIX: &str = "model-blocks/v1"; + +pub struct S3BlockStore { + s3: S3Client, + bucket: String, +} + +impl S3BlockStore { + /// Create a client against the configured endpoint: the same + /// AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY / AWS_ENDPOINT_URL_S3 + /// environment the Tigris repo backend reads. + pub async fn new(bucket: &str) -> anyhow::Result { + let config = aws_config::load_defaults(aws_config::BehaviorVersion::latest()).await; + Ok(Self { + s3: S3Client::new(&config), + bucket: bucket.to_string(), + }) + } + + /// Build a client pointed at an arbitrary endpoint for tests. The + /// production constructor reads the endpoint from the environment, which + /// a test cannot steer; retries are disabled so a dead endpoint fails + /// fast instead of backing off for seconds. + #[cfg(test)] + fn for_testing(bucket: &str, endpoint: &str, timeout: std::time::Duration) -> Self { + use aws_sdk_s3::config::retry::RetryConfig; + use aws_sdk_s3::config::timeout::TimeoutConfig; + use aws_sdk_s3::config::{Credentials, Region}; + + let config = aws_sdk_s3::config::Config::builder() + .endpoint_url(endpoint) + .credentials_provider(Credentials::new("test", "test", None, None, "test")) + .region(Region::new("auto")) + .retry_config(RetryConfig::disabled()) + .timeout_config(TimeoutConfig::builder().operation_timeout(timeout).build()) + .behavior_version_latest() + .build(); + Self { + s3: S3Client::from_conf(config), + bucket: bucket.to_string(), + } + } + + fn key(id: &BlockId) -> String { + let s = id.as_str(); + format!("{KEY_PREFIX}/{}/{s}", &s[..2]) + } + + fn backend(err: E) -> BlockStoreError { + BlockStoreError::Backend(err.to_string()) + } + + async fn upload_one_part( + &self, + key: &str, + upload_id: &str, + number: i32, + bytes: Vec, + ) -> Result { + let out = self + .s3 + .upload_part() + .bucket(&self.bucket) + .key(key) + .upload_id(upload_id) + .part_number(number) + .body(ByteStream::from(bytes)) + .send() + .await + .map_err(Self::backend)?; + Ok(CompletedPart::builder() + .part_number(number) + .e_tag(out.e_tag().unwrap_or_default()) + .build()) + } + + /// Upload the bytes accumulated in `part_buf` as the next part. + async fn flush_part( + &self, + key: &str, + upload_id: &str, + part_buf: &mut Vec, + parts: &mut Vec, + ) -> Result<(), BlockStoreError> { + let staged = std::mem::take(part_buf); + let number = (parts.len() + 1) as i32; + parts.push(self.upload_one_part(key, upload_id, number, staged).await?); + Ok(()) + } + + async fn abort(&self, key: &str, upload_id: &str) { + if let Err(e) = self + .s3 + .abort_multipart_upload() + .bucket(&self.bucket) + .key(key) + .upload_id(upload_id) + .send() + .await + { + // An abort that fails leaves orphaned parts, which cost storage + // but never serve: the object is invisible until a complete + // commits, and only the upload's owner knows the parts' hashes. + tracing::warn!(key = %key, "multipart abort failed: {e}"); + } + } + + /// Classify a failed `CompleteMultipartUpload`: does the failure PROVE + /// the object was not committed? Only a refusal the server took before + /// storing (a 4xx) or a request that never left the client proves that; + /// anything else may have committed while the answer was lost, and the + /// honest answer is `Unknown`. Same rule as `git::tigris`'s dispatch + /// classifier, including the cautious fallback for the `#[non_exhaustive]` + /// error enum. + fn classify_complete_failure(err: &SdkError) -> BlockStoreError { + if let Some(status) = err.raw_response().map(|r| r.status().as_u16()) { + return if (400..500).contains(&status) { + Self::backend(format!("complete refused with status {status}")) + } else { + BlockStoreError::Unknown + }; + } + match err { + SdkError::ConstructionFailure(_) => Self::backend(err.to_string()), + _ => BlockStoreError::Unknown, + } + } +} + +#[async_trait] +impl BlockStore for S3BlockStore { + async fn put_verified( + &self, + id: &BlockId, + mut body: BoxStream<'static, io::Result>, + max_len: u64, + ) -> Result { + let key = Self::key(id); + if self.size(id).await?.is_some() { + return Ok(PutOutcome::AlreadyPresent); + } + + // A zero-length body cannot be a multipart upload, but it can be a + // valid (if useless) object under the empty-sha id. + let upload = self + .s3 + .create_multipart_upload() + .bucket(&self.bucket) + .key(&key) + .send() + .await + .map_err(Self::backend)?; + let upload_id = upload.upload_id().unwrap_or_default().to_string(); + + let mut hasher = Sha256::new(); + let mut len: u64 = 0; + let mut part_buf: Vec = Vec::with_capacity(PART_SIZE); + let mut parts: Vec = Vec::new(); + + // `part_buf` is taken and replaced on each flush, so only one part is + // ever resident, never the whole body. + let result: Result<(), BlockStoreError> = async { + while let Some(chunk) = body.next().await { + let chunk = chunk?; + len = len.saturating_add(chunk.len() as u64); + if len > max_len { + return Err(BlockStoreError::TooLarge(max_len)); + } + hasher.update(&chunk); + part_buf.extend_from_slice(&chunk); + if part_buf.len() >= PART_SIZE { + self.flush_part(&key, &upload_id, &mut part_buf, &mut parts) + .await?; + } + } + Ok(()) + } + .await; + + if let Err(e) = result { + self.abort(&key, &upload_id).await; + return Err(e); + } + if hex::encode(hasher.finalize()) != id.as_str() { + self.abort(&key, &upload_id).await; + return Err(BlockStoreError::DigestMismatch); + } + if !part_buf.is_empty() { + if let Err(e) = self + .flush_part(&key, &upload_id, &mut part_buf, &mut parts) + .await + { + self.abort(&key, &upload_id).await; + return Err(e); + } + } + if parts.is_empty() { + // len == 0 and the hash matched, meaning id is the empty-sha. + self.abort(&key, &upload_id).await; + self.s3 + .put_object() + .bucket(&self.bucket) + .key(&key) + .body(ByteStream::from_static(b"")) + .send() + .await + .map_err(|e| { + if e.raw_response().is_none() { + BlockStoreError::Unknown + } else { + Self::backend(e) + } + })?; + return Ok(PutOutcome::Stored); + } + + let completed = CompletedMultipartUpload::builder() + .set_parts(Some(parts)) + .build(); + match self + .s3 + .complete_multipart_upload() + .bucket(&self.bucket) + .key(&key) + .upload_id(&upload_id) + .multipart_upload(completed) + .send() + .await + { + Ok(_) => Ok(PutOutcome::Stored), + Err(e) => { + let verdict = Self::classify_complete_failure(&e); + if !matches!(verdict, BlockStoreError::Unknown) { + self.abort(&key, &upload_id).await; + } + Err(verdict) + } + } + } + + async fn get_range( + &self, + id: &BlockId, + start: u64, + end_exclusive: u64, + ) -> Result>, BlockStoreError> { + // Bounds come from a HEAD, not from the GET's range answer: S3 + // clamps an over-long end to the object size instead of failing, + // which would silently return fewer bytes than fs does. + let size = match self.size(id).await? { + Some(size) => size, + None => return Err(BlockStoreError::NotFound), + }; + if start > end_exclusive || end_exclusive > size { + return Err(BlockStoreError::OutOfRange); + } + if start == end_exclusive { + return Ok(futures::stream::empty().boxed()); + } + let out = self + .s3 + .get_object() + .bucket(&self.bucket) + .key(Self::key(id)) + .range(format!("bytes={start}-{}", end_exclusive - 1)) + .send() + .await + .map_err(|e| { + if e.as_service_error().is_some_and(|s| s.is_no_such_key()) + || e.raw_response().map(|r| r.status().as_u16()) == Some(404) + { + return BlockStoreError::NotFound; + } + // S3 answers an unsatisfiable range with 416 InvalidRange. + if e.raw_response().map(|r| r.status().as_u16()) == Some(416) { + return BlockStoreError::OutOfRange; + } + Self::backend(e) + })?; + Ok(tokio_util::io::ReaderStream::new(out.body.into_async_read()).boxed()) + } + + async fn size(&self, id: &BlockId) -> Result, BlockStoreError> { + match self + .s3 + .head_object() + .bucket(&self.bucket) + .key(Self::key(id)) + .send() + .await + { + Ok(out) => Ok(out.content_length().map(|l| l as u64)), + Err(e) => { + let status = e.raw_response().map(|r| r.status().as_u16()); + if e.as_service_error().is_some_and(|s| s.is_not_found()) || status == Some(404) { + Ok(None) + } else { + Err(Self::backend(e)) + } + } + } + } + + async fn delete(&self, id: &BlockId) -> Result<(), BlockStoreError> { + self.s3 + .delete_object() + .bucket(&self.bucket) + .key(Self::key(id)) + .send() + .await + .map_err(Self::backend)?; + Ok(()) + } + + fn backend_name(&self) -> &'static str { + "s3" + } +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::body::Bytes as AxBytes; + use axum::extract::{Query, State}; + use axum::http::{HeaderMap, StatusCode}; + use axum::response::Response; + use axum::routing::{delete, get, head, post, put}; + use axum::Router; + use sha2::Digest; + use std::collections::HashMap; + use std::sync::{Arc, Mutex}; + + /// A minimal in-memory S3 mock: just enough of the API surface the + /// backend calls (multipart create/part/complete/abort, put, get, head, + /// delete). No signature check; the SDK client is built with test + /// credentials and the point of the mock is protocol shape, not auth. + #[derive(Default)] + struct Objects { + map: Mutex>>, + parts: Mutex>>, + /// When set, the complete endpoint stores the object and then hangs + /// past the client's operation timeout, simulating a commit whose + /// answer was lost. + hang_on_complete: Mutex, + } + + async fn head_handler( + State(st): State>, + axum::extract::Path((_bucket, key)): axum::extract::Path<(String, String)>, + ) -> Response { + let objects = st.map.lock().unwrap(); + match objects.get(&key) { + Some(body) => Response::builder() + .status(StatusCode::OK) + .header("content-length", body.len().to_string()) + .body(axum::body::Body::empty()) + .unwrap(), + None => Response::builder() + .status(StatusCode::NOT_FOUND) + .body(axum::body::Body::empty()) + .unwrap(), + } + } + + async fn get_handler( + State(st): State>, + axum::extract::Path((_bucket, key)): axum::extract::Path<(String, String)>, + headers: HeaderMap, + ) -> Response { + let objects = st.map.lock().unwrap(); + let Some(body) = objects.get(&key) else { + return Response::builder() + .status(StatusCode::NOT_FOUND) + .body(axum::body::Body::from( + "NoSuchKey", + )) + .unwrap(); + }; + if let Some(range) = headers.get("range").and_then(|v| v.to_str().ok()) { + // bytes=- + let spec = range.trim_start_matches("bytes="); + let (a, b) = spec.split_once('-').unwrap(); + let (start, end): (usize, usize) = (a.parse().unwrap(), b.parse().unwrap()); + if end >= body.len() { + return Response::builder() + .status(StatusCode::BAD_REQUEST) + .body(axum::body::Body::from( + "InvalidRange", + )) + .unwrap(); + } + return Response::builder() + .status(StatusCode::PARTIAL_CONTENT) + .body(axum::body::Body::from(body[start..=end].to_vec())) + .unwrap(); + } + Response::builder() + .status(StatusCode::OK) + .body(axum::body::Body::from(body.clone())) + .unwrap() + } + + async fn put_handler( + State(st): State>, + axum::extract::Path((_bucket, key)): axum::extract::Path<(String, String)>, + Query(q): Query>, + body: AxBytes, + ) -> Response { + if let (Some(part), Some(_)) = ( + q.get("partNumber").and_then(|p| p.parse::().ok()), + q.get("uploadId"), + ) { + st.parts.lock().unwrap().insert((key, part), body.to_vec()); + return Response::builder() + .status(StatusCode::OK) + .header("etag", format!("etag-{part}")) + .body(axum::body::Body::empty()) + .unwrap(); + } + // Plain put (used by the zero-length object path). + st.map.lock().unwrap().insert(key, body.to_vec()); + Response::builder() + .status(StatusCode::OK) + .header("etag", "etag-0") + .body(axum::body::Body::empty()) + .unwrap() + } + + async fn post_handler( + State(st): State>, + axum::extract::Path((_bucket, key)): axum::extract::Path<(String, String)>, + Query(q): Query>, + body: AxBytes, + ) -> Response { + if q.contains_key("uploads") { + return Response::builder() + .status(StatusCode::OK) + .body(axum::body::Body::from( + "u-1", + )) + .unwrap(); + } + if q.contains_key("uploadId") { + // Assemble the object's parts in number order from the complete + // request's part list. + let text = String::from_utf8_lossy(&body); + let mut numbers: Vec = text + .match_indices("") + .map(|(i, _)| { + let rest = &text[i + 12..]; + rest[..rest.find('<').unwrap()].parse().unwrap() + }) + .collect(); + numbers.sort_unstable(); + let mut assembled = Vec::new(); + { + let parts = st.parts.lock().unwrap(); + for n in &numbers { + if let Some(p) = parts.get(&(key.clone(), *n)) { + assembled.extend_from_slice(p); + } + } + } + st.map.lock().unwrap().insert(key.clone(), assembled); + st.parts.lock().unwrap().retain(|(k, _), _| k != &key); + if *st.hang_on_complete.lock().unwrap() { + // The object is stored; the answer never arrives. + futures::future::pending::<()>().await; + } + return Response::builder() + .status(StatusCode::OK) + .body(axum::body::Body::from( + "final", + )) + .unwrap(); + } + Response::builder() + .status(StatusCode::BAD_REQUEST) + .body(axum::body::Body::empty()) + .unwrap() + } + + async fn delete_handler( + State(st): State>, + axum::extract::Path((_bucket, key)): axum::extract::Path<(String, String)>, + Query(q): Query>, + ) -> Response { + if q.contains_key("uploadId") { + st.parts.lock().unwrap().retain(|(k, _), _| k != &key); + } else { + st.map.lock().unwrap().remove(&key); + } + Response::builder() + .status(StatusCode::NO_CONTENT) + .body(axum::body::Body::empty()) + .unwrap() + } + + /// Spawn the mock on an ephemeral port; returns (base_url, state). + async fn spawn_mock(hang_on_complete: bool) -> (String, Arc) { + let st = Arc::new(Objects { + hang_on_complete: Mutex::new(hang_on_complete), + ..Objects::default() + }); + let app = Router::new() + .route("/{bucket}/{*key}", get(get_handler)) + .route("/{bucket}/{*key}", put(put_handler)) + .route("/{bucket}/{*key}", post(post_handler)) + .route("/{bucket}/{*key}", head(head_handler)) + .route("/{bucket}/{*key}", delete(delete_handler)) + .with_state(st.clone()); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + (format!("http://{addr}"), st) + } + + fn id_of(bytes: &[u8]) -> BlockId { + BlockId::parse(&hex::encode(Sha256::digest(bytes))).unwrap() + } + + fn stream(bytes: Vec) -> BoxStream<'static, io::Result> { + futures::stream::iter(vec![Ok(Bytes::from(bytes))]).boxed() + } + + async fn collect(mut s: BoxStream<'static, io::Result>) -> Vec { + let mut out = Vec::new(); + while let Some(c) = s.next().await { + out.extend_from_slice(&c.unwrap()); + } + out + } + + fn store(url: &str) -> S3BlockStore { + S3BlockStore::for_testing("blocks", url, Duration::from_secs(10)) + } + + #[tokio::test] + async fn put_size_get_delete_roundtrip() { + let (url, _st) = spawn_mock(false).await; + let s = store(&url); + let body: Vec = (0..200_000u32).map(|i| (i % 251) as u8).collect(); + let id = id_of(&body); + + assert_eq!( + s.put_verified(&id, stream(body.clone()), u64::MAX) + .await + .unwrap(), + PutOutcome::Stored + ); + assert_eq!(s.size(&id).await.unwrap(), Some(body.len() as u64)); + assert_eq!(s.backend_name(), "s3"); + assert_eq!( + s.put_verified(&id, stream(body.clone()), u64::MAX) + .await + .unwrap(), + PutOutcome::AlreadyPresent + ); + + let got = collect(s.get_range(&id, 1000, 2000).await.unwrap()).await; + assert_eq!(got, body[1000..2000]); + assert!(matches!( + s.get_range(&id, 0, body.len() as u64 + 1) + .await + .map(|_| ()) + .unwrap_err(), + BlockStoreError::OutOfRange | BlockStoreError::Backend(_) + )); + + s.delete(&id).await.unwrap(); + assert_eq!(s.size(&id).await.unwrap(), None); + } + + #[tokio::test] + async fn range_edge_contract_matches_the_trait() { + let (url, _st) = spawn_mock(false).await; + let s = store(&url); + crate::blockstore::parity::get_range_edge_contract(&s).await; + } + + #[tokio::test] + async fn digest_mismatch_aborts_and_stores_nothing() { + let (url, st) = spawn_mock(false).await; + let s = store(&url); + let id = id_of(b"expected"); + let err = s + .put_verified(&id, stream(b"different".to_vec()), u64::MAX) + .await + .unwrap_err(); + assert!(matches!(err, BlockStoreError::DigestMismatch)); + assert_eq!(s.size(&id).await.unwrap(), None); + // The abort removed the staged parts. + assert!(st.parts.lock().unwrap().is_empty()); + } + + #[tokio::test] + async fn over_cap_aborts_while_streaming() { + let (url, _st) = spawn_mock(false).await; + let s = store(&url); + let body = vec![7u8; 4096]; + let id = id_of(&body); + let err = s.put_verified(&id, stream(body), 1024).await.unwrap_err(); + assert!(matches!(err, BlockStoreError::TooLarge(1024))); + assert_eq!(s.size(&id).await.unwrap(), None); + } + + /// A complete request that commits server-side but whose answer is lost + /// must report Unknown, and a later HEAD shows the object is there. The + /// cautious arm exists so a caller never treats a possibly-stored object + /// as definitely absent. + #[tokio::test] + async fn lost_complete_answer_reports_unknown_then_head_resolves() { + let (url, _st) = spawn_mock(true).await; + let s = S3BlockStore::for_testing("blocks", &url, Duration::from_millis(300)); + let body = vec![9u8; 1024]; + let id = id_of(&body); + let err = s + .put_verified(&id, stream(body.clone()), u64::MAX) + .await + .unwrap_err(); + assert!(matches!(err, BlockStoreError::Unknown), "got {err:?}"); + // The write actually committed server-side; a HEAD sees it. + assert_eq!( + store(&url).size(&id).await.unwrap(), + Some(body.len() as u64) + ); + } +} diff --git a/crates/twigpine-node/src/config.rs b/crates/twigpine-node/src/config.rs index 728ec717..bb56a020 100644 --- a/crates/twigpine-node/src/config.rs +++ b/crates/twigpine-node/src/config.rs @@ -316,6 +316,79 @@ pub struct Config { #[arg(long, env = "TWIGPINE_MAX_PACK_BYTES", default_value_t = 2_147_483_648)] pub max_pack_bytes: usize, + /// Enable open-weight model hosting: signed release manifests on public + /// repos, with weights stored as content-addressed blocks outside git. + /// Off by default; when off the node behaves exactly as before. + #[arg(long, env = "TWIGPINE_MODEL_HOSTING_ENABLED", default_value_t = false)] + pub model_hosting_enabled: bool, + + /// Per-identity storage quota for claimed model blocks, in bytes. + /// Default: 256 GiB, enough for a 70B-class bf16 model. + #[arg( + long, + env = "TWIGPINE_MODEL_QUOTA_BYTES", + default_value_t = 274_877_906_944 + )] + pub model_quota_bytes: u64, + + /// Total model bytes this node will hold, in bytes: claimed blocks plus + /// blocks still sitting in open upload sessions. Default: 1 TiB. + #[arg( + long, + env = "TWIGPINE_MODEL_NODE_QUOTA_BYTES", + default_value_t = 1_099_511_627_776 + )] + pub model_node_quota_bytes: u64, + + /// Seconds an uploaded block may sit unclaimed (referenced by no accepted + /// release) before the sweep deletes it. Must exceed the session TTL so an + /// in-progress upload is never swept under its publisher. Default: 86400. + #[arg( + long, + env = "TWIGPINE_MODEL_UNCLAIMED_TTL_SECS", + default_value_t = 86_400 + )] + pub model_unclaimed_ttl_secs: u64, + + /// Base lifetime of an upload session, in seconds. The expiry slides: each + /// accepted block renews it, so a slow upload that keeps making progress + /// never loses its session. Default: 43200 (12 h). + #[arg( + long, + env = "TWIGPINE_MODEL_SESSION_TTL_SECS", + default_value_t = 43_200 + )] + pub model_session_ttl_secs: u64, + + /// Cap, in bytes, on the sum of one identity's open upload sessions. + /// Unset means "use the per-identity quota". + #[arg(long, env = "TWIGPINE_MODEL_MAX_SESSION_BYTES")] + pub model_max_session_bytes: Option, + + /// Directory where model content blocks are stored when the filesystem + /// backend is in use. Sibling of the repos directory under the node data + /// dir by default. + #[arg( + long, + env = "TWIGPINE_BLOCKSTORE_DIR", + default_value = "./data/model-blocks", + global = true + )] + pub blockstore_dir: PathBuf, + + /// S3-compatible bucket for model content blocks. Empty means the + /// filesystem backend at TWIGPINE_BLOCKSTORE_DIR. Uses the same endpoint + /// and credential environment the Tigris bucket uses + /// (AWS_ENDPOINT_URL_S3, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY), so a + /// node already backed by Tigris needs only a bucket name here. + #[arg( + long, + env = "TWIGPINE_BLOCKSTORE_S3_BUCKET", + default_value = "", + global = true + )] + pub blockstore_s3_bucket: String, + /// Per-client-IP rate limit for `POST /api/v1/sync/trigger`, in requests per /// hour. `/sync/trigger` requires a signature and drives an O(peers) outbound /// fan-out per call, so it gets a tight bucket. `0` disables. Default: 60. @@ -1036,6 +1109,14 @@ impl Config { format!("{}:{}", self.host, self.port) } + /// Effective cap on the sum of one identity's open model upload sessions: + /// the explicit cap when set, else the per-identity quota (open sessions + /// may never exceed what the identity could claim). + pub fn effective_model_max_session_bytes(&self) -> u64 { + self.model_max_session_bytes + .unwrap_or(self.model_quota_bytes) + } + /// Resolve ~ in key_path pub fn resolved_key_path(&self) -> PathBuf { if !self.key_path_given { @@ -1316,6 +1397,53 @@ pub(crate) fn validate_web_url(raw: &str) -> Result<(), String> { mod tests { use super::*; + #[test] + fn model_hosting_is_off_by_default_with_expected_budgets() { + let config = Config::parse_from(["twigpine-node"]); + assert!(!config.model_hosting_enabled); + assert_eq!(config.model_quota_bytes, 256 << 30); + assert_eq!(config.model_node_quota_bytes, 1 << 40); + assert_eq!(config.model_unclaimed_ttl_secs, 86_400); + assert_eq!(config.model_session_ttl_secs, 43_200); + assert_eq!(config.model_max_session_bytes, None); + assert_eq!( + config.effective_model_max_session_bytes(), + config.model_quota_bytes + ); + assert_eq!(config.blockstore_dir, PathBuf::from("./data/model-blocks")); + assert_eq!(config.blockstore_s3_bucket, ""); + } + + #[test] + fn model_hosting_flags_parse() { + let config = Config::parse_from([ + "twigpine-node", + "--model-hosting-enabled", + "--model-quota-bytes", + "1024", + "--model-node-quota-bytes", + "4096", + "--model-unclaimed-ttl-secs", + "60", + "--model-session-ttl-secs", + "30", + "--model-max-session-bytes", + "2048", + "--blockstore-dir", + "/var/lib/node/blocks", + "--blockstore-s3-bucket", + "model-blocks", + ]); + assert!(config.model_hosting_enabled); + assert_eq!(config.model_quota_bytes, 1024); + assert_eq!(config.model_node_quota_bytes, 4096); + assert_eq!(config.model_unclaimed_ttl_secs, 60); + assert_eq!(config.model_session_ttl_secs, 30); + assert_eq!(config.effective_model_max_session_bytes(), 2048); + assert_eq!(config.blockstore_dir, PathBuf::from("/var/lib/node/blocks")); + assert_eq!(config.blockstore_s3_bucket, "model-blocks"); + } + #[test] fn upstream_mirror_worker_is_default_off_and_bounded() { let mut config = Config::parse_from(["twigpine-node"]); diff --git a/crates/twigpine-node/src/db/mod.rs b/crates/twigpine-node/src/db/mod.rs index 87903f2b..486a3391 100644 --- a/crates/twigpine-node/src/db/mod.rs +++ b/crates/twigpine-node/src/db/mod.rs @@ -2714,6 +2714,112 @@ const MIGRATIONS: &[Migration] = &[ "CREATE INDEX IF NOT EXISTS idx_repo_push_events_ref ON repo_push_events(repo_id, ref_name, seq DESC)", ], }, + Migration { + // v49: model hosting schema. Nothing reads or writes these tables yet; + // later work mounts upload sessions, release claims, and the + // durability outbox on top of them. Timestamps stay TEXT (RFC 3339), + // the convention every table above follows. + version: 49, + name: "model_hosting_schema", + stmts: &[ + // An upload session is a bearer capability under which a publisher + // streams content-addressed blocks. `bytes_received` bounds what an + // open session holds; `expires_at` slides forward on each accepted + // block so a slow upload that keeps making progress never loses + // its session. + r#"CREATE TABLE IF NOT EXISTS model_upload_sessions ( + id TEXT NOT NULL PRIMARY KEY, + owner_did TEXT NOT NULL, + repo_id TEXT NOT NULL REFERENCES repos(id), + created_at TEXT NOT NULL, + expires_at TEXT NOT NULL, + bytes_received BIGINT NOT NULL DEFAULT 0, + state TEXT NOT NULL CHECK (state IN ('open', 'closed', 'expired')) + )"#, + // The expiry sweep walks open sessions whose deadline passed. + "CREATE INDEX IF NOT EXISTS idx_model_upload_sessions_expiry ON model_upload_sessions (state, expires_at)", + // Per-identity open-session byte totals are summed by owner. + "CREATE INDEX IF NOT EXISTS idx_model_upload_sessions_owner ON model_upload_sessions (owner_did, state)", + // A block is a sha256-addressed run of packed chunks. Its row + // existing is the node's record that the bytes are stored; + // `unreferenced_since` starts the grace clock once no release or + // open session references it. + r#"CREATE TABLE IF NOT EXISTS model_blocks ( + sha256 TEXT NOT NULL PRIMARY KEY CHECK (sha256 ~ '^[0-9a-f]{64}$'), + size BIGINT NOT NULL CHECK (size > 0), + created_at TEXT NOT NULL, + unreferenced_since TEXT + )"#, + // The sweep walks blocks whose grace clock is running. + "CREATE INDEX IF NOT EXISTS idx_model_blocks_unreferenced ON model_blocks (unreferenced_since) WHERE unreferenced_since IS NOT NULL", + r#"CREATE TABLE IF NOT EXISTS model_session_blocks ( + session_id TEXT NOT NULL REFERENCES model_upload_sessions(id), + block_sha256 TEXT NOT NULL REFERENCES model_blocks(sha256), + PRIMARY KEY (session_id, block_sha256) + )"#, + // Reverse membership: the sweep asks "does any session still hold + // this block?" per block, not per session. + "CREATE INDEX IF NOT EXISTS idx_model_session_blocks_block ON model_session_blocks (block_sha256)", + // A release is a signed manifest commit claimed against a repo. + // Unique on (repo, commit): the commit sha is the release id. The + // claim signature material is persisted the way status_claims + // persists it, so a replayed claim is a no-op instead of a second + // quota charge. + r#"CREATE TABLE IF NOT EXISTS model_releases ( + id BIGSERIAL PRIMARY KEY, + repo_id TEXT NOT NULL REFERENCES repos(id), + commit_sha TEXT NOT NULL, + branch TEXT NOT NULL, + publisher_did TEXT NOT NULL, + state TEXT NOT NULL CHECK (state IN ('verifying', 'published', 'rejected', 'withdrawn')), + manifest_sha256 TEXT NOT NULL, + reconstruction_sha256 TEXT, + failure_reason TEXT, + claim_signature TEXT NOT NULL, + claim_signature_input TEXT NOT NULL, + claim_signing_string TEXT NOT NULL, + claim_body BYTEA NOT NULL, + created_at TEXT NOT NULL, + published_at TEXT, + UNIQUE (repo_id, commit_sha) + )"#, + "CREATE INDEX IF NOT EXISTS idx_model_releases_publisher ON model_releases (publisher_did, state)", + // The read surface resolves a branch to its newest published + // release. + "CREATE INDEX IF NOT EXISTS idx_model_releases_repo_branch ON model_releases (repo_id, branch, state, id DESC)", + r#"CREATE TABLE IF NOT EXISTS model_release_blocks ( + release_id BIGINT NOT NULL REFERENCES model_releases(id), + block_sha256 TEXT NOT NULL REFERENCES model_blocks(sha256), + PRIMARY KEY (release_id, block_sha256) + )"#, + // Reverse membership: the sweeper asks "does any release still + // reference this block?", and quota is summed over the distinct + // blocks a DID's live releases reference. + "CREATE INDEX IF NOT EXISTS idx_model_release_blocks_block ON model_release_blocks (block_sha256)", + r#"CREATE TABLE IF NOT EXISTS model_quota_usage ( + did TEXT NOT NULL PRIMARY KEY, + bytes_used BIGINT NOT NULL DEFAULT 0 CHECK (bytes_used >= 0) + )"#, + // Durable outbox for pin and anchor work. One row per (release, + // kind, object), claimed and retried by a worker; provider_cid + // records the returned CID separately from the sha256 that keys + // the object, so the content address stays the resolver key. + r#"CREATE TABLE IF NOT EXISTS model_durability_jobs ( + id BIGSERIAL PRIMARY KEY, + release_id BIGINT NOT NULL REFERENCES model_releases(id), + kind TEXT NOT NULL CHECK (kind IN ('pin_block', 'pin_manifest', 'pin_reconstruction', 'pin_block_map', 'anchor')), + object_key TEXT NOT NULL, + state TEXT NOT NULL CHECK (state IN ('pending', 'done', 'failed')), + attempts INTEGER NOT NULL DEFAULT 0, + last_error TEXT, + provider_cid TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + UNIQUE (release_id, kind, object_key) + )"#, + "CREATE INDEX IF NOT EXISTS idx_model_durability_jobs_pending ON model_durability_jobs (state, created_at)", + ], + }, ]; // ── HTTP-signature replay ledger ────────────────────────────────────────────── @@ -14288,6 +14394,162 @@ mod migration_tests { .unwrap(); assert_eq!(cid, "QmLocalGenuine"); } + + // ── Model hosting schema (v49) ───────────────────────────────────────── + + const V49: i64 = 49; + + const MODEL_TABLES: &[&str] = &[ + "model_upload_sessions", + "model_blocks", + "model_session_blocks", + "model_releases", + "model_release_blocks", + "model_quota_usage", + "model_durability_jobs", + ]; + + async fn model_fixture_repo(db: &super::Db, id: &str) { + sqlx::query( + "INSERT INTO repos (id, name, owner_did, is_public, default_branch, + created_at, updated_at, disk_path) + VALUES ($1, $2, 'did:key:zOwner', TRUE, 'main', + '2026-11-19T00:00:00+00:00', '2026-11-19T00:00:00+00:00', $3)", + ) + .bind(id) + .bind(format!("model-{id}")) + .bind(format!("/srv/repos/{id}.git")) + .execute(&db.pool) + .await + .unwrap(); + } + + async fn insert_model_release(db: &super::Db, repo_id: &str, commit: &str) { + sqlx::query( + "INSERT INTO model_releases + (repo_id, commit_sha, branch, publisher_did, state, + manifest_sha256, claim_signature, claim_signature_input, + claim_signing_string, claim_body, created_at) + VALUES ($1, $2, 'main', 'did:key:zOwner', 'verifying', + $3, 'sig', 'input', 'signing', '\\x00', '2026-11-19T00:00:00+00:00')", + ) + .bind(repo_id) + .bind(commit) + .bind("c".repeat(64)) + .execute(&db.pool) + .await + .unwrap(); + } + + /// A fresh database runs the whole chain and lands every model table. + #[sqlx::test] + async fn migration_v49_fresh_chain_creates_model_tables(pool: sqlx::PgPool) { + let db = super::Db::for_testing(pool); + db.migrate().await.unwrap(); + + for table in MODEL_TABLES { + assert!(table_exists(&db, table).await, "{table} missing"); + } + } + + /// A node already at the previous version gains the same tables through + /// the runner, not just a fresh chain. Rolling back by dropping the v49 + /// objects keeps the test honest if the statements were ever appended to + /// an already-applied entry. + #[sqlx::test] + async fn migration_v49_upgrade_path_adds_model_tables(pool: sqlx::PgPool) { + let db = super::Db::for_testing(pool); + db.migrate().await.unwrap(); + + for table in MODEL_TABLES.iter().rev() { + sqlx::query(&format!("DROP TABLE IF EXISTS {table}")) + .execute(&db.pool) + .await + .unwrap(); + } + sqlx::query("DELETE FROM schema_migrations WHERE version >= $1") + .bind(V49) + .execute(&db.pool) + .await + .unwrap(); + + db.migrate().await.unwrap(); + + for table in MODEL_TABLES { + assert!( + table_exists(&db, table).await, + "{table} missing after upgrade on an existing node" + ); + } + let recorded: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM schema_migrations WHERE version = $1") + .bind(V49) + .fetch_one(&db.pool) + .await + .unwrap(); + assert_eq!(recorded, 1, "v49 must be recorded as applied"); + + // Idempotent re-run. + db.migrate().await.unwrap(); + } + + /// The release id is (repo, commit): a second claim of the same commit on + /// the same repo is a replay the schema refuses. + #[sqlx::test] + async fn model_releases_rejects_duplicate_repo_commit(pool: sqlx::PgPool) { + let db = super::Db::for_testing(pool); + db.migrate().await.unwrap(); + model_fixture_repo(&db, "repo-model").await; + + insert_model_release(&db, "repo-model", SHA_A).await; + let second = sqlx::query( + "INSERT INTO model_releases + (repo_id, commit_sha, branch, publisher_did, state, + manifest_sha256, claim_signature, claim_signature_input, + claim_signing_string, claim_body, created_at) + VALUES ('repo-model', $1, 'main', 'did:key:zOwner', 'verifying', + $2, 'sig', 'input', 'signing', '\\x00', '2026-11-19T00:00:00+00:00')", + ) + .bind(SHA_A) + .bind("d".repeat(64)) + .execute(&db.pool) + .await; + assert!( + second.is_err(), + "a duplicate (repo, commit) release must violate the unique constraint" + ); + + // A different commit on the same repo is a different release. + insert_model_release(&db, "repo-model", SHA_B).await; + } + + /// The block table accepts only a 64-lowercase-hex content address: the + /// id is derived from the bytes everywhere it is used, so a malformed id + /// in the table would corrupt every reference to it. + #[sqlx::test] + async fn model_blocks_rejects_malformed_sha(pool: sqlx::PgPool) { + let db = super::Db::for_testing(pool); + db.migrate().await.unwrap(); + + for bad in ["nothex", &"A".repeat(64), &"g".repeat(64), &"e".repeat(63)] { + let result = sqlx::query( + "INSERT INTO model_blocks (sha256, size, created_at) + VALUES ($1, 10, '2026-11-19T00:00:00+00:00')", + ) + .bind(bad) + .execute(&db.pool) + .await; + assert!(result.is_err(), "block id {bad:?} must be rejected"); + } + sqlx::query( + "INSERT INTO model_blocks (sha256, size, created_at) + VALUES ($1, 10, '2026-11-19T00:00:00+00:00')", + ) + .bind("e".repeat(64)) + .execute(&db.pool) + .await + .unwrap(); + } } #[cfg(test)] diff --git a/crates/twigpine-node/src/lib.rs b/crates/twigpine-node/src/lib.rs index 2173aa00..d068d669 100644 --- a/crates/twigpine-node/src/lib.rs +++ b/crates/twigpine-node/src/lib.rs @@ -9,6 +9,7 @@ pub mod api; mod arweave; mod arweave_v2; pub mod auth; +pub mod blockstore; mod bootstrap; mod cert; pub mod config; @@ -402,6 +403,20 @@ pub async fn run() -> Result<()> { None }; + // Content-addressed model block storage. Built only when model hosting is + // enabled; a misconfigured backend is fatal (fail closed, like purge-spam's + // object store), because falling back to local disk would silently move a + // deployment's blocks somewhere they were never meant to live. + let block_store = crate::blockstore::build(&config) + .await + .context("model block storage failed to initialize")?; + if let Some(ref store) = block_store { + info!( + backend = store.backend_name(), + "model block storage enabled" + ); + } + // Advisory-lock connections come from their own pool: a write guard pins one // for its whole lifetime, and sharing the application pool would let a push // burst starve ordinary request handlers. @@ -621,6 +636,7 @@ pub async fn run() -> Result<()> { 200_000, ), git_bin: "git".to_string(), + block_store, }; // One-time re-encryption of any webhook HMAC secrets still resting as diff --git a/crates/twigpine-node/src/state.rs b/crates/twigpine-node/src/state.rs index 9deaef6c..6925d7f3 100644 --- a/crates/twigpine-node/src/state.rs +++ b/crates/twigpine-node/src/state.rs @@ -383,6 +383,9 @@ pub struct AppState { /// process-group teardown in handler tests without mutating the process-global /// PATH (#174). pub git_bin: String, + /// Content-addressed model block store. `None` unless model hosting is + /// enabled, so every read site must handle the flag-off shape. + pub block_store: Option>, } impl AppState { diff --git a/crates/twigpine-node/src/test_harness.rs b/crates/twigpine-node/src/test_harness.rs index 45e03484..21ec3d8d 100644 --- a/crates/twigpine-node/src/test_harness.rs +++ b/crates/twigpine-node/src/test_harness.rs @@ -228,6 +228,7 @@ fn build_state(db: Arc, pool: PgPool, repos_dir: PathBuf) -> AppS 16, ), git_bin: "git".to_string(), + block_store: None, } } diff --git a/crates/twigpine-node/src/test_support.rs b/crates/twigpine-node/src/test_support.rs index 15f44283..702f2155 100644 --- a/crates/twigpine-node/src/test_support.rs +++ b/crates/twigpine-node/src/test_support.rs @@ -156,6 +156,7 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { ), task_read_rate_limiter: RateLimiter::new(1200, Duration::from_secs(3600)), git_bin: "git".to_string(), + block_store: None, } } diff --git a/docs/RUN-A-NODE.md b/docs/RUN-A-NODE.md index 01fb488c..60792c8e 100644 --- a/docs/RUN-A-NODE.md +++ b/docs/RUN-A-NODE.md @@ -282,6 +282,24 @@ short lifetimes reissued often are safer than one long-lived grant. --- +## Model hosting (experimental) + +Off by default and not yet usable end to end: the node ships the storage and +schema only, with no publish or read routes mounted yet. When it lands, a model +is a public repo whose releases are signed manifests, and weights are stored as +content-addressed blocks outside git. + +- `TWIGPINE_MODEL_HOSTING_ENABLED=false` - the master switch. +- `TWIGPINE_MODEL_QUOTA_BYTES` - per-identity quota for claimed blocks (default 256 GiB). +- `TWIGPINE_MODEL_NODE_QUOTA_BYTES` - total model bytes this node will hold (default 1 TiB). +- `TWIGPINE_MODEL_UNCLAIMED_TTL_SECS` - grace period before unclaimed blocks are swept (default 86400). +- `TWIGPINE_MODEL_SESSION_TTL_SECS` - base upload-session lifetime, renewed by each accepted block (default 43200). +- `TWIGPINE_MODEL_MAX_SESSION_BYTES` - optional cap on one identity's open sessions (defaults to the quota). +- `TWIGPINE_BLOCKSTORE_DIR` - where blocks live on disk (default `./data/model-blocks`). +- `TWIGPINE_BLOCKSTORE_S3_BUCKET` - set to store blocks in S3 instead; reuses the same `AWS_*` endpoint and credentials as the Tigris repo backend. + +--- + ## Operational checklist | Concern | Recommendation |