Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -464,3 +464,25 @@ ICAPTCHA_PUBKEY=
ICAPTCHA_REQUIRED_LEVEL=3
# Base URL of the iCaptcha service.
ICAPTCHA_URL=https://icaptcha.gitlawb.com

# ── Model hosting (experimental, off by default) ──────────────────────────
# Signed model releases on public repos, with weights stored as
# content-addressed blocks outside git. Off by default; not yet usable end to
# end: this ships the storage and schema only, no publish or read routes yet.
TWIGPINE_MODEL_HOSTING_ENABLED=false
# Per-identity quota for claimed model blocks, in bytes (default 256 GiB).
TWIGPINE_MODEL_QUOTA_BYTES=274877906944
# Total model bytes this node will hold, claimed plus open sessions (1 TiB).
TWIGPINE_MODEL_NODE_QUOTA_BYTES=1099511627776
# Seconds an uploaded block may sit unclaimed before the sweep deletes it.
TWIGPINE_MODEL_UNCLAIMED_TTL_SECS=86400
# Base lifetime of an upload session; each accepted block renews it.
TWIGPINE_MODEL_SESSION_TTL_SECS=43200
# Optional cap on the sum of one identity's open upload sessions, in bytes.
# Unset means "use TWIGPINE_MODEL_QUOTA_BYTES".
#TWIGPINE_MODEL_MAX_SESSION_BYTES=
# Where model blocks live. Defaults to a model-blocks dir beside the data dir.
# Leave the bucket empty to use the filesystem; set it to store blocks in S3
# using the same AWS_* endpoint/credentials as the Tigris repo backend.
TWIGPINE_BLOCKSTORE_DIR=./data/model-blocks
TWIGPINE_BLOCKSTORE_S3_BUCKET=
13 changes: 10 additions & 3 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -431,6 +431,7 @@ Important node settings:
| `TWIGPINE_AUTO_SYNC` | Enable automatic sync from known peers. |
| `TWIGPINE_RECONCILIATION_SWEEP` | Enable the hourly durability sweep that re-pins/backstops missing objects (default `true`; disabled when no IPFS/Pinata backend is configured). Public pin repair runs against any configured backend (IPFS, Pinata, or both). Encrypted recovery repair requires local IPFS (`TWIGPINE_IPFS_API`); Pinata-only nodes reconcile public pins only. |
| `TWIGPINE_MAX_PACK_BYTES` | Max git pack body size for smart-HTTP routes. |
| `TWIGPINE_MODEL_HOSTING_ENABLED` | Enable experimental model hosting (signed releases, block storage). Off by default; see `docs/RUN-A-NODE.md` for the quota and block-store knobs. |
| `TWIGPINE_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, or `info/refs` advertisement may run before it is aborted (504). Default 600. Also bounds the withheld-blob classification walk (on both the upload-pack serve and receive-pack replication paths) and the push-side pin-candidate discovery (`rev-list` / `cat-file`), each reaped via process-group teardown at the deadline. On the path-scoped upload-pack path the classification walk and the pack serve share ONE deadline, so this value bounds their combined duration rather than granting each stage a full budget: a walk that consumes it leaves the serve nothing and the clone gets a 504. Serving large path-scoped repos may therefore need a higher value than they did when each stage was budgeted separately. Accepted range is 1 to 3153600000 (100 years), since the node derives deadlines from this value and a larger one cannot be represented. |
| `TWIGPINE_GIT_ACQUIRE_TIMEOUT_SECS` | Max seconds the storage-acquisition phase (Tigris HEAD/GET, push advisory-lock) of a served git op may run before the request is shed with a 503, separate from the git-run timeout. The concurrency permit is released on expiry so a stalled backend cannot pin the pool. Default 30. |
| `TWIGPINE_MAX_CONCURRENT_GIT_OPS` | Max concurrent served git READ ops (upload-pack and its `info/refs` advertisement) across all callers; over-cap sheds a 503 + Retry-After. Anonymous reads draw from this pool, so pair it with `TWIGPINE_MAX_CONCURRENT_READS_PER_CALLER`. Pushes and the receive-pack advertisement have their own pools, so a read flood cannot shed an authenticated push. Default 128. |
Expand Down
5 changes: 5 additions & 0 deletions crates/twigpine-core/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,11 @@ curve25519-dalek = "4"
crypto_box = { version = "0.9", features = ["std", "chacha20"] }
chacha20poly1305 = "0.10"

# Content-defined chunking (FastCDC, the 2020 algorithm variant) for the
# model-block format. Streaming over a reader, so memory stays proportional
# to the block size rather than the file size.
fastcdc = "5"

[dev-dependencies]
tokio = { workspace = true }
# Not optional here: the redirect matrix must run under a bare
Expand Down
1 change: 1 addition & 0 deletions crates/twigpine-core/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ pub mod identity;
pub mod identity_path;
pub mod legacy_env;
pub mod legacy_headers;
pub mod model;
pub mod node_denial;
// `url` is the one dependency here that drags a tail (idna, then the icu
// crates), and twigpine-core is allowlisted to stay embeddable. Every client that
Expand Down
139 changes: 139 additions & 0 deletions crates/twigpine-core/src/model/chunking.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,139 @@
//! Content-defined chunking and block packing.
//!
//! Chunking uses FastCDC (the 2020 algorithm variant) at 8 / 64 / 128 KiB
//! min / avg / max. Packing into blocks lives in `reconstruction::pack_file`,
//! which seals at the 64 MiB target or end of stream.

use std::io::Read;

use fastcdc::v2020::StreamCDC;

use super::sha256_hex;

pub const CHUNK_MIN: usize = 8 * 1024;
pub const CHUNK_AVG: usize = 64 * 1024;
pub const CHUNK_MAX: usize = 128 * 1024;
pub const BLOCK_TARGET: usize = 64 * 1024 * 1024;

/// One content-defined chunk of a file. `data` is present while packing; the
/// id is the chunk's sha256, which is how the chunk dedups across files and
/// releases.
#[derive(Debug)]
pub struct Chunk {
pub sha256: String,
pub len: usize,
pub offset: u64,
pub data: Vec<u8>,
}

/// A sealed block: a run of chunks and its content id.
#[derive(Debug)]
pub struct Block {
pub sha256: String,
pub chunks: Vec<Chunk>,
pub data: Vec<u8>,
}

/// Stream a reader into content-defined chunks. Memory stays bounded by the
/// maximum chunk size, never the file size.
pub fn chunk_reader<R: Read>(reader: R) -> impl Iterator<Item = std::io::Result<Chunk>> {
StreamCDC::new(reader, CHUNK_MIN, CHUNK_AVG, CHUNK_MAX).map(|r| {
let c = r?;
Ok(Chunk {
sha256: sha256_hex(&c.data),
len: c.data.len(),
offset: c.offset,
data: c.data,
})
})
}

#[cfg(test)]
mod tests {
use super::*;
use crate::model::gen_bytes;
use std::io::Cursor;

const SEED: u64 = 0x9e3779b97f4a7c15;

fn chunk_all(bytes: &[u8]) -> Vec<Chunk> {
chunk_reader(Cursor::new(bytes.to_vec()))
.collect::<Result<_, _>>()
.unwrap()
}

#[test]
fn chunking_is_deterministic() {
let bytes = gen_bytes(3 * 1024 * 1024, SEED);
let a = chunk_all(&bytes);
let b = chunk_all(&bytes);
assert_eq!(a.len(), b.len());
for (x, y) in a.iter().zip(b.iter()) {
assert_eq!(x.sha256, y.sha256);
assert_eq!(x.len, y.len);
assert_eq!(x.offset, y.offset);
}
}

#[test]
fn chunk_sizes_stay_within_bounds() {
let bytes = gen_bytes(10 * 1024 * 1024, SEED);
let chunks = chunk_all(&bytes);
assert!(chunks.len() > 10);
for (i, c) in chunks.iter().enumerate() {
assert!(c.len <= CHUNK_MAX, "chunk {i} exceeds max");
if i + 1 < chunks.len() {
// The final chunk may end before the min when the input runs out.
assert!(c.len >= CHUNK_MIN, "chunk {i} below min");
}
}
}

#[test]
fn a_small_edit_changes_only_early_chunks() {
let base = gen_bytes(20 * 1024 * 1024, SEED);
let mut edited = vec![0u8; 100];
edited.extend_from_slice(&base);

let base_ids: Vec<String> = chunk_all(&base).iter().map(|c| c.sha256.clone()).collect();
let edited_ids: Vec<String> = chunk_all(&edited)
.iter()
.map(|c| c.sha256.clone())
.collect();

let base_set: std::collections::HashSet<&String> = base_ids.iter().collect();
let changed = edited_ids
.iter()
.filter(|id| !base_set.contains(id))
.count();
// 100 inserted bytes move the first few boundaries; the vast majority
// of a 20 MiB file still chunks identically.
assert!(
changed <= 4,
"{changed} changed chunks for a 100-byte prefix insert"
);
}

#[test]
fn chunk_data_never_buffers_the_file() {
// The contract this format depends on: a reader is consumed a chunk
// at a time. Guard it by chunking a reader that would panic if read
// past its claimed end before the iterator asks for more.
struct Stingy {
left: usize,
}
impl Read for Stingy {
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
let n = buf.len().min(self.left);
self.left -= n;
buf[..n].fill(1);
Ok(n)
}
}
let mut it = chunk_reader(Stingy {
left: 3 * CHUNK_MAX,
});
let first = it.next().unwrap().unwrap();
assert!(first.len <= CHUNK_MAX);
}
}
Loading
Loading