diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 323ec55b..447f58df 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,46 +50,13 @@ jobs: version: ${{ env.FOUNDRY_VERSION }} - name: Run unit tests run: cargo test --lib --features test-utils - # Before the e2e suite, deliberately. These are fast and deterministic, and the e2e - # suite flakes on hosted runners for transport reasons that have nothing to do with - # storage. A failing step aborts the job, so anything sequenced after a flaky one - # never reports, which is how these ran on no platform at all for a whole run. - - name: Prove the migration returns disk to the filesystem + - name: Kill a node mid-write and check what survived shell: bash run: | set -euo pipefail - cargo test --test migration_reclaims_disk --features test-utils -- --nocapture --test-threads=1 2>&1 | tee /tmp/reclaims_disk.log - # A target whose required features are not passed is skipped with a - # warning and a zero exit, so a harness can stop running without anyone - # noticing. This is what makes that loud. - grep -qE 'test result: ok\. [1-9]' /tmp/reclaims_disk.log \ - || { echo 'reclaims_disk ran no tests'; exit 1; } - - name: Kill a node mid-migration and check what survived - shell: bash - run: | - set -euo pipefail - cargo test --test migration_crash_safety --features test-utils -- --test-threads=1 2>&1 | tee /tmp/crash_safety.log - # A target whose required features are not passed is skipped with a - # warning and a zero exit, so a harness can stop running without anyone - # noticing. This is what makes that loud. + cargo test --test chunk_store_crash_safety --features test-utils -- --test-threads=1 2>&1 | tee /tmp/crash_safety.log grep -qE 'test result: ok\. [1-9]' /tmp/crash_safety.log \ - || { echo 'crash_safety ran no tests'; exit 1; } - - name: Several nodes migrating on one disk - shell: bash - run: | - set -euo pipefail - cargo test --test migration_shared_volume --features test-utils 2>&1 | tee /tmp/shared_volume.log - # A target whose required features are not passed is skipped with a - # warning and a zero exit, so a harness can stop running without anyone - # noticing. This is what makes that loud. - grep -qE 'test result: ok\. [1-9]' /tmp/shared_volume.log \ - || { echo 'shared_volume ran no tests'; exit 1; } - # Linux only. This one plants a hundred thousand files to measure what a restart - # costs, and the answer it is after is a fleet answer, where every node is Linux. - # The scan itself reads names and nothing else, which is not a platform-specific - # path, and opening a store is covered on all three by the unit tests. Planting that - # many files on the Windows runner would cost minutes of every run to re-measure - # something no node will ever do there. + || { echo 'chunk_store_crash_safety ran no tests'; exit 1; } - name: Startup scan, index memory and inode cost at scale if: runner.os == 'Linux' shell: bash @@ -109,15 +76,7 @@ jobs: run: cargo test --test poc_audit_handler_live --features test-utils - name: Run bootstrap-stall PoC regression marker run: cargo test --test poc_bootstrap_stall --features test-utils - - name: Shutdown waits for writes whose caller has gone - run: cargo test --test poc_shutdown_lmdb_drain --features test-utils - # Runs the storage tests against real ext4, XFS and btrfs rather than whatever the - # runner provides. Deliberately NOT named durability: killing a process and reopening - # the same mounted filesystem keeps the page cache, so this exercises each filesystem's - # syscall, locking, rename and delete behaviour, not its behaviour under power loss. - # That still needs block-device fault injection or a real machine, and remains a fleet - # gate. filesystems: name: Storage on ${{ matrix.fs }} runs-on: ubuntu-latest @@ -139,52 +98,32 @@ jobs: # on whatever the runner happens to give us. ext4 is what most of the fleet is # on; XFS and btrfs are the two the design reasons about separately, btrfs # because it has been observed reordering writes around a rename. - # 3 GiB is ample: these tests use tens of MiB. The scale harness, which is - # the one that needs room, is not in this job. + # 3 GiB is ample: these tests use tens of MiB. truncate -s 3G /tmp/${{ matrix.fs }}.img mkfs.${{ matrix.fs }} -q /tmp/${{ matrix.fs }}.img sudo mkdir -p /mnt/antfs sudo mount -o loop /tmp/${{ matrix.fs }}.img /mnt/antfs sudo chown "$USER" /mnt/antfs df -hT /mnt/antfs - # TMPDIR is what `TempDir::new` uses, so this is what puts the test data on the - # mounted filesystem rather than on the runner's root. - - name: The migration returns disk on ${{ matrix.fs }} + # TMPDIR is what `TempDir::new` uses, so this puts every temporary store these tests + # build on the mounted filesystem rather than on the runner's root. + # + # These used to be the migration harnesses. The migration is gone, and what is left + # worth asking of a filesystem is what the store itself does on it: publish a chunk + # through a temporary and a rename, flush the directory, unlink it again, and rebuild + # an index from the names afterwards. The storage tests do all of that, and running + # them here is what keeps ext4, XFS and btrfs covered now that the harnesses that used + # to cover them have been deleted. They do not measure free space before and after; + # the harness that did was about the migration and went with it. + - name: Storage behaviour on ${{ matrix.fs }} env: TMPDIR: /mnt/antfs shell: bash run: | set -euo pipefail - cargo test --test migration_reclaims_disk --features test-utils -- --nocapture --test-threads=1 2>&1 | tee /tmp/reclaims_disk.log - # A target whose required features are not passed is skipped with a - # warning and a zero exit, so a harness can stop running without anyone - # noticing. This is what makes that loud. - grep -qE 'test result: ok\. [1-9]' /tmp/reclaims_disk.log \ - || { echo 'reclaims_disk ran no tests'; exit 1; } - - name: A node killed mid-write on ${{ matrix.fs }} loses nothing - env: - TMPDIR: /mnt/antfs - shell: bash - run: | - set -euo pipefail - cargo test --test migration_crash_safety --features test-utils -- --test-threads=1 2>&1 | tee /tmp/crash_safety.log - # A target whose required features are not passed is skipped with a - # warning and a zero exit, so a harness can stop running without anyone - # noticing. This is what makes that loud. - grep -qE 'test result: ok\. [1-9]' /tmp/crash_safety.log \ - || { echo 'crash_safety ran no tests'; exit 1; } - - name: Several nodes on one ${{ matrix.fs }} volume - env: - TMPDIR: /mnt/antfs - shell: bash - run: | - set -euo pipefail - cargo test --test migration_shared_volume --features test-utils 2>&1 | tee /tmp/shared_volume.log - # A target whose required features are not passed is skipped with a - # warning and a zero exit, so a harness can stop running without anyone - # noticing. This is what makes that loud. - grep -qE 'test result: ok\. [1-9]' /tmp/shared_volume.log \ - || { echo 'shared_volume ran no tests'; exit 1; } + cargo test --lib --features test-utils storage:: 2>&1 | tee /tmp/storage.log + grep -qE 'test result: ok\. [1-9]' /tmp/storage.log \ + || { echo 'the storage tests ran nothing'; exit 1; } doc: name: Documentation @@ -212,7 +151,8 @@ jobs: - name: Build release (no logging) run: cargo build --release --no-default-features # The crash harness drives the store through a failpoint that parks the process - # forever on an environment variable. It is compiled only under `test-utils`, which + # forever on an environment variable, and the store's own tests use the same one. It + # is compiled only under `test-utils`, which # is not a default feature and is not passed by the release workflow, so a shipped # binary does not contain it. This proves that rather than trusting it: the variable # name is a string literal, so it survives into the binary whenever the code that diff --git a/Cargo.lock b/Cargo.lock index c9ae4b1d..c12b1b37 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -831,13 +831,13 @@ dependencies = [ "mimalloc", "objc2", "objc2-foundation", - "page_size", "parking_lot", "postcard", "proptest", "rand 0.8.6", "reqwest", "rmp-serde", + "rustix", "saorsa-core", "saorsa-pqc", "self-replace", diff --git a/Cargo.toml b/Cargo.toml index 2888c86a..e45c50e5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -56,7 +56,8 @@ xor_name = "5" lru = "0.16.3" parking_lot = "0.12" # Efficient mutex for cache -# Storage - LMDB via heed for content-addressed chunk store +# LMDB via heed for the paid-key list. The chunk store is one file per chunk and does +# not use it. heed = "0.22" blake3 = "1" @@ -109,8 +110,6 @@ sha2 = "0.10" # Cross-platform file locking for upgrade caches fs2 = "0.4" -# System page size (for LMDB map alignment during resize) -page_size = "0.6" # Protocol serialization postcard = { version = "1.1.3", features = ["use-std"] } @@ -118,6 +117,11 @@ bao = "0.13.1" [target.'cfg(unix)'.dependencies] libc = "0.2" +# Safe wrappers for `openat`/`unlinkat`, so the storage-migration cleanup can delete relative +# to a directory handle it opened `O_NOFOLLOW` rather than by re-resolving a path it has +# already checked. Adds nothing to the build: `tempfile`, a direct dependency, already pulls +# this exact crate and version in with the `fs` feature on. +rustix = { version = "1", features = ["fs"] } [target.'cfg(windows)'.dependencies] self-replace = "1" @@ -132,20 +136,6 @@ proptest = "1" alloy = { version = "1", features = ["node-bindings"] } serial_test = "3" -# Proves the migration returns disk to the filesystem, which is the claim the whole -# change exists to make good. Needs the test-only migration-state accessor. -[[test]] -name = "migration_reclaims_disk" -path = "tests/migration_reclaims_disk.rs" -required-features = ["test-utils"] - -# Kills a real child process part-way through writing and migrating, then checks what -# survived. The automatable half of the power-loss gate. -[[test]] -name = "migration_crash_safety" -path = "tests/migration_crash_safety.rs" -required-features = ["test-utils"] - # Startup scan time, index memory and inode cost at scale. Regression gates, not # benchmarks; ANT_SCALE_KEYS raises the count for a deliberate larger run. [[test]] @@ -153,11 +143,11 @@ name = "storage_scale" path = "tests/storage_scale.rs" required-features = ["test-utils"] -# Several nodes migrating on one disk: the volume lock, and that each finishes with its -# own chunks and only its own. +# A process killed part-way through a write: the store is whole or absent, never +# half-indexed, and what the interrupted write left behind is swept. [[test]] -name = "migration_shared_volume" -path = "tests/migration_shared_volume.rs" +name = "chunk_store_crash_safety" +path = "tests/chunk_store_crash_safety.rs" required-features = ["test-utils"] # E2E test infrastructure (run with --features test-utils) @@ -176,7 +166,7 @@ path = "tests/poc_commitment_audit_attacks.rs" required-features = ["test-utils"] # Live responder-handler tests for the v12 audit. Use -# LmdbStorageConfig::test_default(), gated on test-utils. +# ChunkStoreConfig::test_default(), gated on test-utils. [[test]] name = "poc_audit_handler_live" path = "tests/poc_audit_handler_live.rs" @@ -197,14 +187,6 @@ name = "poc_price_floor_live" path = "tests/poc_price_floor_live.rs" required-features = ["test-utils"] -# Shutdown/LMDB-drain regression: `ReplicationEngine::shutdown()` must not -# return while a detached LMDB blocking op is still running. Uses the -# test-only storage put gate, so it requires the test-utils feature. -[[test]] -name = "poc_shutdown_lmdb_drain" -path = "tests/poc_shutdown_lmdb_drain.rs" -required-features = ["test-utils"] - [features] default = ["logging"] # Enable tracing/logging infrastructure. diff --git a/config/production.toml b/config/production.toml index 72e82112..f1f1ac61 100644 --- a/config/production.toml +++ b/config/production.toml @@ -46,62 +46,14 @@ enabled = true # Verify content hash on read verify_on_read = true -# Maximum size in GiB of the legacy LMDB store, while one still exists -# (0 = derive it from available disk). Retired along with LMDB itself. -db_size_gb = 0 - -# --- Moving off the legacy LMDB chunk store --- -# -# Chunks are now one file each, under {root_dir}/chunks/. A node that still has a -# chunks.mdb copies it into files in the background, then deletes it whole, which is the -# only moment LMDB's disk comes back. -# -# The two release-level switches (whether to delete the old store, and whether audits -# still penalise) belong to the build, not to this file, so they are deliberately absent. -[storage.migration] -# Run the copier. Turning this off leaves both stores in place forever and never -# returns the old store's disk. -enabled = true - -# Also write new chunks to the legacy store while it exists, so a fleet rollback to an -# older build cannot lose a chunk uploaded during the migration. -dual_write_legacy = true - -# Allow a node that cannot fit its chunks to give up the ones it is furthest from. -# -# Whatever this is set to, a chunk is only ever given up when the node is near the back of -# its group for it, its close group has received the node's reduced commitment, AND all but -# one of that group has cryptographically proven it holds a copy. A node that cannot show -# all three keeps both stores and asks for more disk. Turn this off if you would rather add -# disk than have the node give anything up at all. -allow_shed = true - -# Hours after this build first starts before a node may give anything up, so peers on -# older builds have upgraded and stopped penalising it for doing so. -shed_hold_hours = 72 - -# Hours between one migration wave opening and the next. -# -# A close group is split into waves so only two of its members give chunks up at a time. -# If all seven went together none could prove to the others that a copy survived, and the -# group would deadlock waiting on each other. A node with room to copy everything does not -# wait for a wave: it is never unable to serve, so it is not part of that problem. -wave_hours = 24 - -# Hours between a node committing to what it will keep and deleting the old store. -# Never shorter than 4: that is what the answerability window needs. -retire_delay_hours = 4 - -# Free space, in MiB, the copier leaves untouched on top of disk_reserve_mb. -copier_slack_mb = 2048 - -# Copy rate ceiling, in MiB/s. Keep it modest: an unthrottled copier competing with the -# audit responder for disk turns a storage migration into an audit incident. -copier_throttle_mib_per_sec = 32 - # --- Upgrade --- [upgrade] -enabled = false +# There is no `enabled` setting, and there never was one. This file used to carry +# `enabled = false`, which serde ignored: `UpgradeConfig` has no such field, so every node +# reading this was self-upgrading while its own configuration appeared to say otherwise. +# Removed rather than annotated, because a line that does nothing is worse than an absent one +# — and this one said the opposite of the truth about the mechanism a release rollback would +# have to go through. channel = "stable" check_interval_hours = 1 github_repo = "WithAutonomi/ant-node" diff --git a/deploy/scripts/spawn-nodes.sh b/deploy/scripts/spawn-nodes.sh index 445bca62..49f741c4 100755 --- a/deploy/scripts/spawn-nodes.sh +++ b/deploy/scripts/spawn-nodes.sh @@ -68,15 +68,6 @@ fi # Create directories mkdir -p "$BASE_DIR" "$LOG_DIR" -# The per-volume migration lock. Every node on this host shares it and nothing else, so -# they can serialise their copies off LMDB without being able to reach each other's data. -# It needs its own directory because PrivateTmp=true below gives each unit a /tmp of its -# own, and the node's default lock location is in there: without this every node takes a -# lock nobody else can see, all of them start copying at once, and the host runs out of -# space with several half-finished migrations on it. -LOCK_DIR="${BASE_DIR%/*}/migration" -mkdir -p "$LOCK_DIR" - # Create ant user if not exists if ! id -u ant &>/dev/null; then useradd -r -s /bin/false ant || true @@ -99,8 +90,6 @@ for i in $(seq 0 $((NODE_COUNT - 1))); do # Create node directory mkdir -p "$NODE_DIR" chown ant:ant "$NODE_DIR" - chown ant:ant "$LOCK_DIR" - chmod 0750 "$LOCK_DIR" # Create systemd service cat > "/etc/systemd/system/$SERVICE_NAME.service" <> /etc/security/limits.conf diff --git a/docs/adr/ADR-0014-file-based-chunk-store-and-lmdb-retirement.md b/docs/adr/ADR-0014-file-based-chunk-store-and-lmdb-retirement.md index c0631a2f..5acacf4d 100644 --- a/docs/adr/ADR-0014-file-based-chunk-store-and-lmdb-retirement.md +++ b/docs/adr/ADR-0014-file-based-chunk-store-and-lmdb-retirement.md @@ -370,6 +370,50 @@ belief carry its own expiry — the directory carries its mark, the proof carrie saw, the write carries its note — rather than to check again and hope the check is close enough to the act. +## Amendment: see what the neighbours say, and stop the migration slashing anyone + +Shipped as a patch on top of this record's release, which had already merged. It changes no +on-disk format and writes no migration state: a node part-way through re-reads its marker, its +first-start time and its remaining keys and carries on. That is asserted by a test rather than +argued. + +**A node holding data no longer clears its own commitments.** `storage_empty` asked whether +there was anything left to *commit to*, not whether there were any bytes. Once the migration +settles the commitment narrows to the file-backed set, and `all_keys` drops a file marked +suspect, so two nodes that hold data reported empty: one whose disk filled before it could copy +anything, and one whose last readable file went transiently bad. Both took the `clear_all` +branch, which drops every retained root with no answerability window. An auditor holding a root +gossiped minutes earlier then got `UnknownCommitment` — a confirmed failure on the +commitment-bound lane, which is deliberately enforced in every release and is *not* the lane +the migration holds off. The node slashed itself for data it still had, and no switch could +stop it. Those nodes now take `retire_current`: stop advertising, stay answerable until the +gossip TTL lapses, bytes on disk. That is the staged narrowing this migration was designed +around; the predicate is what routes a node into it. + +**Every node says whether it still has an old chunk store, and reads what its neighbours say.** +The state rides the user agent `saorsa-core` already sends with every signed message and keeps +per peer, so this costs no new message, no new field and no protocol version, and the `node/` +prefix that gates DHT membership is preserved. Three states, never folded into two: a directory +that could not be read is not one that is not there. + +What the peer half means, in the fewest words that are all true, because a release decision +rests on it. + +It counts what the peers a node is connected to **announced**, each as of that peer's own last +start. `saorsa-core` copies the user agent when it builds the transport, so a node that finishes +migrating goes on announcing `legacy` until it restarts. + +Two consequences, running in opposite directions, so the tally bounds nothing. A peer announcing +`legacy` may have finished since, so the count can be too high. A node that is offline, or simply +not connected to, is absent from it, so the count can be too low. An all-zero tally proves +nothing on its own either, because a node connected to nobody produces one; the number of peers +seen is what tells that apart. + +So this can surface nodes that have not finished. It cannot establish that none remain, and no +amount of it adds up to that. `outstanding` counts `legacy`, `unknown` and `unreported` +together, because a peer whose disk could not be read and a peer on a build from before this +existed are both as far from finished as `legacy` is. + ## Consequences ### Positive @@ -519,10 +563,10 @@ restored. matching prediction. - The second gates on a soak of the first, plus a verified retirement returning the predicted space. -- The third gates on migration-complete lines across the fleet, refetch backlogs drained, and the - recorded audit failure rate back to its pre-migration baseline. The first release's - observability is - what makes that decidable. +- The third gates on migration-complete lines from the nodes we run, refetch backlogs drained, + and the recorded audit failure rate back to its pre-migration baseline. The observability + added here informs that call; it does not decide it. Nothing here can establish that a node + we neither run nor are connected to has finished. - **How often a short-of-disk node can actually clear the possession gate.** A node whose close group is also short of space will not clear it, will not free its disk, and will tell its operator to add storage. That is the intended answer, but the fleet needs to diff --git a/docs/adr/ADR-0015-remove-the-lmdb-chunk-store.md b/docs/adr/ADR-0015-remove-the-lmdb-chunk-store.md new file mode 100644 index 00000000..cf1ac71d --- /dev/null +++ b/docs/adr/ADR-0015-remove-the-lmdb-chunk-store.md @@ -0,0 +1,613 @@ +# ADR-0015: Remove the LMDB Chunk Store + +- **Status:** Proposed +- **Date:** 2026-08-28 +- **Decision owners:** Anselme Gaeremynck +- **Reviewers:** David Irvine, Chris O'Neil, Mick van der Most van Spijk +- **Supersedes:** none +- **Superseded by:** none +- **Related:** ADR-0014 (one file per chunk, and retiring LMDB), which this completes + +## Context + +Moving chunks off LMDB shipped as three releases, because the penalty for not holding a +close-group chunk is the *auditor's* decision: a node that has to give chunks up cannot stop +its peers punishing it for that. So the peers stopped first. + +1. **First:** suspend that one penalty. +2. **Second:** copy every chunk into a file of its own, then delete `chunks.mdb`. ADR-0014. +3. **Third:** this one. + +ADR-0014 describes the third release as flipping the switch back and nothing more. What +actually has to happen is larger, and two parts of it are decisions rather than clean-up. + +## Decision + +**Do not restore the penalty here.** It stays suspended for one more release, and the +release after this one turns it back on. The reasoning is in *The penalty is restored by the +release after this one* below: a node that was away while the migration ran arrives here +holding a store this build cannot read, and restoring the accusation in the release that +stranded it would slash it for a state it had no chance to leave. + +The switch itself stays. The process-wide atomic, the `ANT_SUSPEND_UNHELD_CHUNK_PENALTY` +override and the startup announcement are not migration machinery: they are one release-level +policy that several audit paths have to obey identically, and the release that restores a +penalty is exactly the one most likely to need it undone in a hurry. Removing them would discard the +cheapest lever at the moment it is most useful. A test now pins the shipped value, because +the existing tests set the switch both ways on purpose and so could never notice which way +it was compiled. + +**Delete the LMDB chunk store and the migration, and keep the name `ChunkStore`.** There is +one store. It is one file per chunk, it lives in `src/storage/chunk_store.rs`, and it is +called `ChunkStore` because that is what it is and what every caller already called it. The +type that used to present two stores as one is gone with the second store. + +`heed` stays in the dependency list. The paid-key list has its own LMDB environment, which +this decision does not touch. + +**A node that still has an unretired `chunks.mdb` starts anyway, and clears up what it can +prove is finished with.** This is the part worth arguing, and two earlier drafts of this +decision got it wrong in opposite directions. + +That draft refused to start, and the reasoning was not silly. The chunks in that environment +are unreachable to this build, but the commitment the node published before the upgrade +*claimed* them, and a commitment stays answerable to its neighbours for three hours +(`GOSSIP_ANSWERABILITY_TTL`, which is `(RETAINED_GOSSIPED_COMMITMENTS + 1)` rotations). The +accusation the first release suspended was "you did not have a chunk you were supposed to +hold"; the commitment-bound subtree audit was never suspended in any release, precisely +because it rests on a signed claim. So a node that starts half-migrated does spend hours +failing audits, at the full weight, on the one lane that always counted. + +Three things make refusing the worse answer anyway. + +**A node that refuses serves nothing.** Not the chunks it cannot read, and not the far larger +number it migrated perfectly well. The bounded harm being avoided is a few hours of trust +penalty; the harm being accepted is the entire node, and not for a few hours. + +**There is no recovery.** The obvious instruction, "put the previous build back and let it +finish", cannot be carried out. `Node::build_upgrade_monitor` is called unconditionally and +`UpgradeConfig` has no field that switches it off, so a node put back on the previous release +polls, finds this one, and takes it. There is nothing to set, in a config file or on the +command line, that stops it. On the deployed unit (`Restart=always`, `RestartSec=10`) a +refusing node is a ten-second restart loop with no health check and no automatic rollback, and +it stays in it until a person intervenes. An instruction nobody can follow is not a +mitigation. + +(How quickly it is taken depends on the deployment. The unit in `deploy/terraform` runs the +node as `ant` under `ProtectSystem=strict` with write access only to its own directory, and +the binary lives in `/usr/local/bin`, so the in-process replacement cannot complete there at +all. That is a separate problem with that unit, not a way to hold a node on an old release: +nothing in the node consents to staying.) + +**The population that reaches here is not the one refusing would protect.** A node that has +been offline long enough to miss the previous release entirely has no retained root at any +peer — the answerability window is three hours and its close group has long since re-placed +what it held — so it takes no *commitment-bound* penalty, which is the lane refusing was +protecting it from. The node that is exposed on that lane is one that was running and +unmigrated when this release landed, and that is exactly the population the previous release's +fleet signal exists to count and wait out. + +It is not penalty-free, and it is worth being exact about what remains rather than rounding it +to nothing. This release does NOT restore the close-group unheld-chunk penalty; that waits for +the release after it, for the reason given below. What it cannot hold off is the +commitment-bound audit, which is a separate lane and is enforced in every release. A node +carrying an old store it cannot read is a node +with that much less disk, and a node short of disk fails to take on the chunks it is +responsible for and is penalised on that lane like any other full node. That is not a penalty +for having migrated badly; it is the ordinary consequence of a full disk, arriving through a +directory that is full of nothing useful. The node is told exactly that, once, by name, and +the remedy is the operator's: add disk, or delete the directory once its contents are known to +be elsewhere. + +Refusing to start would not have spared it either. It would have had the same disk and none of +the service. + +So this build starts, and it clears up **only what is provably finished with**. + +Deleting whatever it finds was the second draft, and it was also wrong. The upgrade monitor +picks the newest eligible release rather than the next one, so a node that was offline +through the previous release arrives here with every chunk it owns in that environment and +nothing at all in the file store. Deleting that is destroying data that may have no other +copy, in order to reclaim disk. + +The evidence that separates the two is the mark the previous release wrote *inside* the +directory before it deleted anything. A directory carrying `RETIRED` is one whose retirement +gates were all satisfied, and it is pure cost. So is an empty one, which is what a cleanup +interrupted between emptying a tombstone and removing it leaves. Those go, and their disk comes +back. + +**Be exact about what those gates were, because "its contents were copied out" is not it** and +an earlier draft of this record said so four times. Retirement cleared a directory on two +different grounds, and only the first is a local copy: + +- every chunk the node was **keeping** was copied into the file store and re-hashed there, byte + for byte, before the mark went down; and +- every chunk the node was **shedding** was proven to be held by its close group — all but one + of them answering a possession challenge with a cryptographic proof, after the reduced + commitment had been delivered — and was then deliberately *not* copied. The pre-retirement + verification pass skips exactly these keys, because re-hashing a chunk the node is giving up + into a file store that is not going to keep it would defeat the point of shedding it. + +So a marked directory can, entirely legitimately, contain bytes that are in no file store on +this node and never will be. The previous release was about to delete those bytes itself; this +one finishes that. The safety argument for them is the close group's proofs, not a local copy, +and it is the network that holds them afterwards. + +| what is on disk | what the cleanup does | what the node then serves | disk back | +|---|---|---|---| +| nothing, or a root that does not exist yet | nothing | whatever it stores from here | n/a | +| a leftover carrying its `RETIRED` mark | remove it in the background, report the space once it is back | its file store, which is all of it | yes | +| a leftover with nothing in it | remove it | its file store | yes | +| a leftover with chunks in it and no mark | **keep it**, name it once | its file store only — **not** what is in that directory | no | +| a leftover whose contents or mark cannot be established | keep it, name it once | its file store only | no | +| a link at either name | keep it, name it once | its file store only | no | +| a name retirement never created | ignore it entirely | its file store | no | + +The rows that keep something are the point, and the third column is the part it would be easy +to round off. **Kept is not the same as available.** There is no LMDB reader in this build, so +a node that keeps a directory keeps its bytes on disk and cannot serve one of them. For a node +that was part-way through the migration that is the tail it had not copied yet; for a node that +skipped the previous release altogether it is everything it holds, and such a node serves only +what it refetches from here on, exactly as a new node would, while its old bytes sit there +costing disk. + +That is worth being plain about because it is the whole price of the decision: the data is +kept so that it is still there to be recovered by hand or by a future build, not because this +release can do anything with it. What the node gets is its own service back; what it does not +get is that disk, which is the honest cost of not being able to prove the directory is safe to +delete. + +Two things are never done, both because a name is not evidence of what is behind it. + +A **link is neither followed nor unlinked**. What is behind it is on storage this node does +not own: following it would delete somebody else's data, and unlinking it would throw away +the only record of where that data went. + +Only the **exact names** the previous release created are considered: `chunks.mdb`, +`chunks.mdb.retired`, and `chunks.mdb.retired.` for `n` in `1..=64` written without a +leading zero. An earlier draft matched by prefix and by "all digits", which would have claimed +a `chunks.mdb.retired-keep-this` an operator put there, and `.007` and `.999999`, which +retirement cannot have produced. The suffix is parsed and written back out so it has to match +itself, and there is a test that plants every near miss. + +The deletion runs on **one** background thread and nothing waits for it. `remove_dir_all` over +a store with millions of files runs for minutes and must not be on the startup path. One +thread rather than one per directory, and they are deleted in turn: the names this release +recognises are the live directory, the unnumbered tombstone and sixty-four numbered ones, so a +root that has been through enough restore cycles can present sixty-six at once, and a thread +each would put sixty-six concurrent recursive deletions on the disk that is also serving +chunks, at the moment a node is starting. Nothing is waiting on them, so doing them in turn +costs nothing that matters. It deletes in place: there is nothing to get out of the way, because the directory holds no chunks and this +build has no code that would read it if it did. An earlier draft renamed first and could run +out of names to rename to, at which point it stopped removing anything at all, permanently. +The space is only reported as returned once the deletion has actually finished, because a +number that is wrong for the next several minutes is worse than no number. + +**The mark has to be the file the previous release wrote**, not merely something at that +name. It is written with `create_new`, so it is always an ordinary file; a directory, a link +or a FIFO wearing the name proves nothing, and this is the answer that authorises deleting +every chunk underneath it. Testing existence alone would let anything at that path clear an +unmigrated store for deletion. + +**The mark is believed, and not re-checked against anything.** It is worth stating as an +assumption rather than leaving it to be inferred, because it is the one that authorises every +deletion here. `RETIRED` records that the *previous release* satisfied both gates above — the +copy-and-re-hash for what it kept, the close-group possession proofs for what it shed. This +build cannot confirm either independently: it cannot re-run a possession challenge for keys it +cannot enumerate, and it has no LMDB reader, so it cannot compare what is in the directory against +what is in the file store, and no cheaper check is available — a file store that opens is not +evidence that it holds any particular key, and counting keys proves nothing about which ones. + +So the mark is taken as final. For every state the previous release can actually produce, that +is correct: it wrote the mark after the copy and the re-hash, and a marked directory in this +release is one whose deletion was interrupted. The state it is wrong for is one no release +produces — an operator restoring an old marked directory alongside a file store that is not the +one it was retired against, or putting a file called `RETIRED` inside an environment by hand. +This release will delete such a directory. That is accepted: the alternative is to keep every +marked leftover for ever, which returns no disk on any node and defeats the release, and the +states that would be protected are ones a person constructed. **An operator restoring a backup +of `chunks.mdb` must not leave the `RETIRED` file in it.** + +**The mark is removed last.** `remove_dir_all` gives no promise about the order it unlinks +things in, and if the mark went before the chunks did and the process stopped there, the next +start would find an unmarked directory with data in it, decide it might be an unmigrated +store, and keep it for good. It is not one, but nothing on disk would say so any more, and +the node would report itself unfinished to the whole network for as long as it lived. So the +contents go, then the mark, then the directory: at every point either the mark is still there +and the next start resumes, or the directory is empty, which is also finished with. + +**None of it happens until the file store has actually opened.** "Finished with" means +retirement's gates were met, and for everything the node kept that means the chunks are in the +file store, which is only true if the file store is there to hold them. +An earlier draft ran this as soon as the root was known, which put the deletion in front of a +constructor that can still fail on an unreadable layout, a directory it cannot create, or a +lock another process has not let go of — and a node that lost both stores that way had +nothing to go back to. Waiting costs nothing in what this node reports: its user agent is +fixed when the transport is built, a moment earlier, but everything removed here is a leftover +the signal already reads as finished with — carrying the mark or being empty is both what +makes it removable and what makes it harmless — so the node announces `files` whether the +deletion has finished, is still running, or has not started. + +Nothing here can fail. `clean_up` returns `()`. Every way a filesystem can disappoint it ends +in a node that runs, a warning that names the directory, and disk that has not come back — +which is a worse place than success and a far better one than a restart loop. + +Nothing here can leave a node with no store: the deletion runs only after the store that +replaced it has opened, and only over directories that provably hold no chunks. A node that +never opens one, because storage is switched off, deletes nothing at all — it has established +nothing about where those chunks went, and it has no use for the disk either. + +Said precisely, because the looser version of it is not true: **nothing here vetoes a start**. +That is not the same as "every node starts". The file store is built before this runs, and a +store that cannot open — an unreadable layout, a directory it cannot create, a lock another +process holds — still stops the node, exactly as it did in the release before this one. What +this release removes is the *other* reason a node could fail to start, the one its first draft +introduced: being refused for what it was found carrying. + +**What may be deleted is decided by the previous release's own classifier**, not by a second +reading of the same directory. The signal ADR-0014 put on the wire reports a node as finished +when nothing under its root is holding chunks or unreadable, and this release is published on +the strength of that count; so the cleanup calls that same function rather than carrying its +own idea of which directories are finished with. Two classifiers could drift, and either +direction is a fault: one would let a node delete a directory it was still reporting as +unfinished, the other would leave it reporting `files` while paying for the disk for ever. +There is a test that stages every shape a root can present and checks both directions of that +correspondence, so a re-introduced private classifier fails there rather than on a fleet. + +**The deletion is made against a directory handle, not against a path.** An earlier draft of +this decision named a path for the checks and named it again for the unlinking, and accepted the +window between them on the ground that whoever could win it already had write access to the data +directory. That reasoning was wrong, and independent review caught it: it covers deleting things +*inside* the data directory and says nothing about the variant that reaches outside. `read_dir` +follows a link. Point that name at a target elsewhere on the disk in the window, and the node +empties the target instead — and the node reaches far more of the filesystem than the actor who +moved the name does, and on many installations runs as root. That is privilege escalation, not +one more way to lose a chunk. + +So on Unix the directory is opened once, `O_NOFOLLOW | O_DIRECTORY`, and every unlink is made +against that handle with `unlinkat`; subdirectories are opened the same way from the same handle +and emptied the same way. A link cannot produce a handle, so no unlink can be redirected through +one. The pattern and the reasoning are already in this codebase: `open_regular` refuses a link +and a FIFO on the handle rather than on the path, for the same class of reason. + +The directory itself still goes by path, and that is safe on its own terms: `rmdir` refuses a +symlink, so a swapped name fails the call rather than following it. + +What is pinned by test is the primitive, not the interleaving, and the difference is worth being +exact about. The race cannot be staged without instrumenting the deleter. What the tests assert +is that a link never yields a handle, and that a *marked* target behind a link is untouched — +marked deliberately, because an unmarked one is refused by the gates even by a deleter that +follows links, so testing against one would pass while proving nothing. Both fail if `O_NOFOLLOW` +is dropped. + +**Two things this still does not close, both stated rather than half-fixed.** Off Unix there is +no `unlinkat`, so that build keeps the path-based deleter and its window; its exposure is what +the previous release's deleter already had. And on either build the empty-directory case cuts +both ways within the data directory: an entry created inside a directory this found empty is +deleted with it, and an entry created after the mark has gone leaves an unmarked directory with +something in it, which every later start then keeps for good. + +### What this release does NOT delete + +The cleanup matches exactly `chunks.mdb`, `chunks.mdb.retired` and `chunks.mdb.retired.` +for `n` in `1..=64`, and nothing else. Two things under the node root are deliberately +outside that set and must stay outside it: + +- **The migration marker.** Small, harmless, and useful evidence if a node's history is ever + in question. +- **Anything else an operator put there.** A prefix match would claim a + `chunks.mdb.retired-keep-this`, which is why names are matched exactly rather than by + prefix. + +That the cleanup does not touch these today is a property of the exact-name match, not an +accident, and it is stated here because a future widening of that match would be a silent +data loss rather than an obvious one. + +### Values over the size ceiling are not chunks, and none is preserved + +This was argued both ways across the three releases, and one of the earlier answers — preserve +them to a sidecar rather than destroy them — was wrong. It is settled here so that a later +reader does not reopen it from the sidecar's remains. + +**A chunk is at most 4 MB.** `MAX_CHUNK_SIZE` has been `4 * 1024 * 1024` since ant-protocol's +first commit, and every released path by which data enters a node from the network enforces it +before anything is stored: the protocol handler on a paid store, and replication on both the +receive and the fetch path. **So no value over the ceiling has ever entered this network as a +chunk.** It does not follow that none can exist on a disk — one can, and the next paragraph says +how — only that anything which does is not a chunk: not data with a copy elsewhere, not data +whose owner is waiting for it, and nothing any peer would accept or serve. + +**The one way such a value could reach a disk was ours.** Not the network's. The bridge's +public `ChunkStore::put` wrote to the legacy environment *first* — `LmdbStorage::put` has no +size ceiling, deliberately, because the store it wrote to was being abandoned and its verdict +was not allowed to refuse chunks the file store had room for — and only then offered the same +bytes to the file store, which refused them for size. The bridge then recorded the key as +legacy-only so the copier would retry it, and the copier's own size arm deleted it. Every +oversized value that can exist on any node came through a local caller of that method during +the bridge period, and through nothing else. + +**This release removes the bridge, so it removes the hole.** There is one store; its `put` +refuses anything over the ceiling, and refuses *before* it writes, so there is no partial state +for a later pass to find and no key recorded anywhere. There is no `LmdbStorage::put` left to +take an unbounded value in the first place. The read path and the repair path refuse over the +ceiling too, so a file planted by hand is refused rather than served. + +**Nothing is preserved and no sidecar is built.** An earlier draft added one, on the reasoning +that no peer can hold a copy of an over-ceiling value and no repair can fetch one, so deleting +it destroys the only copy. Both halves of that are true and the conclusion still does not +follow: there is no valid chunk there to be the only copy *of*. Building somewhere to keep +invalid values would be building for a case this release makes unreachable, and the cost of +doing it was measured — the sidecar was written, and adversarial review found two real defects +inside it, in code that existed only to serve a case that cannot arise. It was cut before it +shipped, and it is not coming back here. + +A legacy directory that happens to contain such a value is kept if it is unmarked and removed if +it carries the mark, exactly like every other directory, and for exactly the same reasons. This +release never looks inside one, so its contents are not a factor in either verdict. + +### The penalty is restored by the release after this one, not by this one + +The original plan had this release delete the old store and restore the close-group storage +penalty together. They are now separated, and the separation is the point. + +The upgrade monitor picks the newest eligible release rather than the next one. A node that was +offline while the migration ran therefore arrives here having never migrated, holding a legacy +store this build cannot read. This release keeps that store rather than deleting it, which is +right for its data, but the node cannot serve those chunks and its close group will notice. +Restoring the accusation in the same release would slash that node for a state it had no chance +to leave, in the release that put it there. + +So this release ships with the penalty still held off, and the release after it restores the +penalty once the fleet has been observed clean for long enough to include the nodes that were +away. That is a one-line change to a build constant, and it costs one extra release to stop the +cleanup and the accusation landing on a stranded node at the same moment. + +What this leaves is one migration-era constant still in the tree after the release that was +meant to remove them all. That is a deliberate trade: the objective this protects is that no +release breaks the fleet, and it outranks the objective that no migration code survives. + +## Consequences + +### Positive + +- One store, one name, and about 5,600 lines of bridge and driver gone. +- The unheld-chunk penalty is still held off, and the release after this one restores it. +- **The cleanup never vetoes a start.** Not the same as "every node starts": the file store + is built first and one that cannot open still stops the node, exactly as before. What is + gone is the other reason, the refusal this release's first draft introduced. +- The per-volume migration lock and its deployment settings go with the migration. + +### Negative / Trade-offs + +- **A node that never finished migrating keeps its old store and does not get that disk + back.** That population is the short-of-disk nodes, and how large it is remains the open + fleet question ADR-0014 records — which is why that release now reports it on the wire, and + why this one is not published until the count is clean. Such a node runs and serves what it + migrated; what it does not do is reclaim the space, which it could not do safely under any + of the three answers considered here. +- **The two halves have very different emergency levers, which is why they are not in the same + release.** Restoring the penalty is a switch, undoable in minutes and from the fleet side. + Deleting the bridge can only be undone by not shipping the release at all — and not even by + rolling the binary back, since the upgrade monitor takes the node forward again. That + asymmetry is the reason the penalty waits for the release after this one rather than riding + along with the deletion, and it is a decision rather than a scheduling accident. +- `ChunkStore` and its module were renamed from `FileStore` and `file_store.rs`. Callers of + the facade did not change, because they already used that name — but **`FileStore`, + `FileStoreConfig`, `StoreLayout`, `VerifyReport` and `LEGACY_ENV_DIR` were exported too**, + and all five are gone from the public API along with the LMDB and migration types. A + downstream crate importing any of them stops compiling, so they belong in the release + notes and not only in this list. + +### Neutral / Operational + +- `ANT_SUSPEND_UNHELD_CHUNK_PENALTY` still works and still logs loudly when it disagrees + with the build. +- `storage.migration` and `storage.db_size_gb` are gone from the configuration. The second + capped a memory map that no longer exists, and a setting that silently does nothing is + worse than one that is absent. Nothing declares `deny_unknown_fields`, so a config file + written by the previous release still loads with both keys in it, which is what stops + every node on the fleet failing to start at once on upgrade. There is a test for that, + because adding that attribute later would look harmless. Worth knowing what that test is: a + hand-written fragment carrying the two removed keys, not a complete file emitted by the + previous release. It proves those two keys are tolerated; it does not prove a whole persisted + config round-trips, and an incompatibility in a field it omits would not be caught by it. +- There is no supported way to make this build read an old chunk store, and no way to stop + it clearing one up. A leftover it declines to touch — a link, an unmarked store with chunks + in it, or one whose state cannot be established — is named in a warning carrying + `migration_event = "legacy_store_left"`, and is the operator's to remove. + +## Validation + +**Proved here.** A marked environment is removed and its disk comes back; so is a marked +tombstone, and so is an empty one. An **unmarked** environment with chunks in it is still +there afterwards, and so is an unmarked tombstone — those two are the data-loss tests, and +they are the ones that would have gone red against the draft that deleted everything. A node +whose root does not exist yet is fine. Four directories that only *look* like leftovers +(`chunks.mdb.retired-keep-this`, `chunks.mdb.backup`, `.65`, `.007`) are all still there, and +the name matcher is asserted directly on both sides. A linked environment is untouched: the +link is still there and what it points at still has its data, which is the pair of mistakes +prefix matching and link following would each have made. + +A directory called `RETIRED` sitting where the mark would be does not authorise anything, and +the store it is in is still there afterwards. The deletion's order is asserted directly: with +a subdirectory made unremovable, the attempt fails and the mark has to still be there, because +that is what lets the next start recognise the directory rather than treat it as unmigrated +forever. + +A value over the ceiling is refused by `put`, and refused *before* anything is written: the +store does not claim it, no file is left under its name, and a restart onto the same directory +finds nothing to index. Addressed to its own bytes on purpose, so the refusal is the size arm +and not the content-address arm — checking the wrong arm would pass while the ceiling was gone. +Mutation-checked: with the size branch deleted the test fails, which is what says it is testing +the ceiling rather than testing that something went wrong. + +Three further properties are pinned that the earlier draft had no test for, because it had no +code for them either. + +**The correspondence with the fleet signal.** A root is staged carrying at least one shape for +every verdict the classifier can return — harmless three ways (a marked live environment, a +marked tombstone, an empty one), holding two (an unmarked tombstone with chunks in it, a link +wearing a tombstone's name), unreadable two (something that is not the mark using the mark's +name, and a plain file wearing a tombstone's name) — plus the near misses that are not in the +name set at all. Each is classified before anything is removed, and afterwards each is asserted +gone exactly when it was called harmless and still there exactly when it was not. Both +directions, so a classifier that drifts either way fails here; mutation-checked by making a link +classify harmless, which the test catches. + +**A marked directory is removed even when nothing was copied into this node's file store**, +staged with no file store at all. That is a shedding node's ordinary state, and pinning it is +what stops somebody later adding a local-copy check that would strand every such directory for +ever. + +**Sixty-six leftovers**, the whole namespace this release accepts, are removed by one start. And +**a root that cannot be listed** has nothing removed from it, which is the case that used to +return silently and left this record promising a warning nothing emitted. + +**What these do NOT prove, said rather than implied.** The sixty-six-leftover test asserts that +all of them go; it does not observe how many threads did it, so it would still pass if the one +sequential worker went back to one per directory. The deletion ORDER — mark last — has its +failure half asserted only on Unix, because staging a part-way failure needs a permission bit +there and an open handle on Windows; off Unix that test proves only that a successful deletion +leaves nothing behind. And the entry-level unreadable case is not staged: the test that makes a +root unlistable exercises `read_dir` itself failing, not one `DirEntry` failing inside a +readable root, so turning that arm from a refusal into a skip would restore the false-green and +stay green here. + +Through `NodeBuilder::build()`: a node with an unmigrated store starts under both +`storage.enabled = true` and `false` and still has its chunks afterwards; a node with a marked +one starts and the leftover goes; and a node whose file store cannot open — staged with a file +where the chunk directory has to be — fails to build with its old store still intact, which is +what pins the deletion behind the replacement. The penalty staying SUSPENDED is pinned by a test +on the shipped constant, which fails if it is flipped — in either direction. The existing tests +set the switch both ways on purpose, so none of them could ever notice which way it was +compiled, which is how a suspension outlives the thing it was suspended for. + +**Deleted, and what replaced it.** ADR-0014's validation section describes four harnesses. +Most of three of them existed to prove the bridge worked: that the disk came back when the +old store was deleted, that a node killed mid-copy lost nothing, and that several nodes on +one disk took turns. There is no bridge left for those to test. The fourth, which measures +what one file per chunk costs at scale, stays. + +Not all of it went, and saying it did was wrong. Two tests inside the crash harness were +never about the bridge: that a process killed mid-publish leaves no chunk the store cannot +serve, and that what an interrupted write leaves behind is swept. Those are about the store's +own publish path, which is now the only one there is, so they matter more after this release +rather than less. They are back as `tests/chunk_store_crash_safety.rs` and run in CI. A third +property, that engine shutdown waits for a detached store write, is named under the gaps +below. + +**The hardening release's two tests, one deleted and one that must keep passing.** The release +that hardened the migration added exactly two, and this release does something different with +each, so both are named rather than left to be noticed in a diff. +`a_store_left_midway_by_the_previous_release_keeps_its_place` proved that an upgrade picks a +half-finished migration up where it left off instead of restarting its clock — it drives +`copy_batch`, `migration_phase`, `legacy_only_keys` and `migration_state`, all of which this +release deletes. It goes with its subject: there is no migration left for an upgrade to +continue, and a test of one cannot be rewritten against a release that has none. +`retiring_keeps_a_pinned_root_answerable_and_clearing_does_not` is the opposite case. It is why +the commitment rotation has no emptiness branch, it touches nothing this release removes, and it +still passes here. **It must go on passing**, because the branch it rules out is exactly the one +an earlier draft of this release was going to reinstate. + +That leaves the loopback filesystem job with nothing to run, and deleting it would quietly +drop ext4, XFS and btrfs coverage of the store itself. It now runs the storage unit tests +against each mounted filesystem instead, which is what still has something to say there: +publishing through a temporary and a rename, flushing, deleting, and rebuilding an index +from the names. + +**Not proved here, and inherited from ADR-0014.** Forced power loss on the five filesystems. +Scale at one and ten million keys. How many short-of-disk nodes can clear the possession +gate. Under ADR-0014 such a node keeps serving from both stores; under this one it starts, +serves what it migrated, keeps what it did not, and never gets that disk back. + +**Coverage this release drops, named rather than lost.** A harness proved that +`ReplicationEngine::shutdown()` waits for a store write whose awaiter was dropped before it +returns. It was written against the old store and went with it. The property is still +current and still claimed by that method's own documentation, and nothing tests it now. It +needs a live P2P node to stage, which is why it is called out here rather than quietly +rewritten in the same change that deleted it. + +**A fleet gate this decision adds.** Before this ships, the fleet has to show that nodes are +actually on the file store, because this is the release that stops any of them going back. +ADR-0014 puts the answer on the wire: every node announces `migration/legacy`, +`migration/files` or `migration/unknown` in the user agent it already sends with every signed +message, and each node counts what it sees around it. The gate is that nothing reports +`legacy` or `unknown` for itself, and that no observed peer reports `legacy`, `unknown`, or +nothing at all, over several consecutive days, against a roster of nodes we expect to hear +from. A peer that reports nothing is running a build from before the signal existed, which is +not evidence of anything having finished; the tally counts it as outstanding for that reason. Silence is not readiness: +a zero from a collector that scraped nothing looks exactly like a zero from a clean fleet, +which is why the count needs a denominator and not just a numerator. + +That gate cannot be made airtight, and it is worth being honest about which part is soft. +Nodes that are offline for the whole window are in nobody's count and come back afterwards; +that is the population this decision knowingly cleans up rather than the one it waits for. + +**Where that gate stands as this is written, which is nowhere yet.** The signal first shipped in +`v0.19.0-beta.1`, published 2026-09-09, so there is under a day of it against a gate that asks +for several consecutive days. More to the point, the beta's own lines do not mean yet what the +gate needs them to mean: the staged migrations have not started, so a node reporting `files` +today is reporting that it has nothing to move rather than that it has finished moving it, and +those two are indistinguishable in the tally. And the population the gate exists for — the +community nodes, and the NTFS hosts where the store's case-folding and directory-entry +behaviour differ from ours — is not reporting to us at all. **No count taken before the staged +migrations run should be read as progress towards this gate**, and this release is not published +on the strength of one. + +## What this release does not fix + +Named rather than implied. None is a regression; each is the state before this change. + +- **A node that kept an unreadable store does not get that disk back**, and being short of disk + is how that costs it: it fails to take on chunks it is responsible for. Note that this + release does not add the unheld-chunk accusation on top — that lane is still suspended here + and returns in the release after. What such a node is exposed to is the commitment-bound + lane, which every release enforces. The remedy is the operator's, and the warning names the + directory. +- **Off Unix the path check and the unlink are not one operation.** They are on Unix, where the + deletion is made against a handle opened `O_NOFOLLOW`; `std` offers no portable `unlinkat`, so + the other platforms keep the path-based deleter and the window the previous release's deleter + also had. +- **On a case-folding filesystem a leftover can be kept that should have gone.** Tombstone names + are matched exactly, as strings, against what `read_dir` reports, while NTFS and a + default-configured APFS compare names case-insensitively. A directory stored as + `CHUNKS.MDB.RETIRED` is the same file to the filesystem and a different string to the matcher, + so it is not recognised and is kept for ever. That is the safe direction — the failure is disk + not returned, never data removed — and it takes an operator having renamed something, since + retirement only ever writes lower case. The live name is unaffected: it is looked up by name + rather than matched from a listing, so the filesystem's own comparison finds it. +- **`deploy/terraform/cloud-init/worker.yml` still cannot start a node**: it passes no rewards + address, which production mode requires. The binary also sits in `/usr/local/bin` while the + unit runs under `ProtectSystem=strict`, so an in-process upgrade cannot replace it. Both + predate this work and belong to whoever owns that deployment. That path is not evidence for + the fleet gate until they are fixed. +- **Off Unix a chunk is published under its final name**, so a power loss can leave a real + chunk name over partial bytes, and a commitment built before anything reads it claims a + chunk the node cannot produce. ADR-0014 states this; the forced power-loss run is still an + open gate. +- **There is no supported way to hold a node on an earlier release.** This is named here + because it is what makes the fleet gate a gate rather than a preference. The on-disk format + rolls back cleanly — the previous release reads the same one-file-per-chunk layout, and a + legacy directory this release kept is one it can still pick up and finish — but the + *operation* does not: `build_upgrade_monitor` is called unconditionally, `UpgradeConfig` has + no field that disables it, and the monitor takes the newest eligible release, so a node put + back on the previous binary is dragged forward again within the hour. Rolling back to + v0.18.1 is worse than unsupported, because chunks this release accepted exist only in the + file store that build does not have. + + The fix is an upgrade-subsystem one — a persisted disable, or a version ceiling — and it is + deliberately not in this release: it changes the mechanism every node uses to take every + release, which is not a change to make in the release that also deletes a store. It belongs + in its own change, with its own evidence. + + What must not be said here, and an earlier draft of this record did say it, is that no state + this release creates would ever have wanted a rollback. A marked directory can hold shed + chunks, which are bytes the previous release proved the close group holds and deliberately did + not copy locally. Deleting them is right and is what that release was about to do — but it is + irreversible on this node, and rolling back would not bring them back either, because the + previous release would have deleted them too. The remedy for those keys was never local: it is + replication fetching them from the peers that proved they hold them. + +## Notes for AI-assisted work + +Drafted with AI assistance. Not to be marked Accepted without human review. diff --git a/src/config.rs b/src/config.rs index 5f11a49d..790bdf1c 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,6 +1,5 @@ //! Configuration for ant-node. -use crate::storage::MigrationConfig; use evmlib::Network as EvmNetwork; use serde::{Deserialize, Serialize}; use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4}; @@ -410,7 +409,7 @@ const fn default_staged_rollout_hours() -> u64 { /// Controls how chunks are stored, including: /// - Whether storage is enabled /// - Content verification on read -/// - Database size limits (auto-scales with available disk by default) +/// - How much free disk to leave unused #[derive(Debug, Clone, Serialize, Deserialize)] pub struct StorageConfig { /// Enable chunk storage. @@ -423,27 +422,12 @@ pub struct StorageConfig { #[serde(default = "default_storage_verify_on_read")] pub verify_on_read: bool, - /// Explicit LMDB database size cap in GiB. - /// - /// When set to 0 (default), the map size is computed automatically from - /// available disk space at startup and grows on demand when the operator - /// adds storage. Set a non-zero value to impose a hard cap. - #[serde(default)] - pub db_size_gb: usize, - /// Minimum free disk space (in MiB) to preserve on the storage partition. /// /// Writes are refused when available space drops below this threshold, /// preventing the node from filling the disk completely. Default: 500 MiB. #[serde(default = "default_disk_reserve_mb")] pub disk_reserve_mb: u64, - - /// Controls for moving this node off the legacy LMDB chunk store. - /// - /// The two release switches inside it are deliberately not serialised: see - /// [`MigrationConfig`]. - #[serde(default)] - pub migration: MigrationConfig, } impl Default for StorageConfig { @@ -451,14 +435,12 @@ impl Default for StorageConfig { Self { enabled: default_storage_enabled(), verify_on_read: default_storage_verify_on_read(), - db_size_gb: 0, disk_reserve_mb: default_disk_reserve_mb(), - migration: MigrationConfig::default(), } } } -/// Default: 500 MiB — matches `DEFAULT_DISK_RESERVE` in `storage::lmdb`. +/// Default: 500 MiB — matches `DEFAULT_DISK_RESERVE` in `storage`. const fn default_disk_reserve_mb() -> u64 { 500 } @@ -608,32 +590,59 @@ fn default_testnet_bootstrap() -> Vec { #[allow(clippy::unwrap_used, clippy::expect_used)] mod tests { - use super::*; - use serial_test::serial; - + /// A config file written by the previous release still loads. + /// + /// The settings that drove the migration off the old chunk store are gone from this + /// build, and so is the database size cap, which configured a memory map that no longer + /// exists. Every node on the fleet has a config file on disk carrying them, written by + /// the release that did the migrating. If those keys made the file unparseable, every + /// one of those nodes would fail to start on upgrade, all at once. + /// + /// Nothing declares `deny_unknown_fields`, so serde ignores them. That is the behaviour + /// this depends on, which makes it worth a test rather than an assumption: adding that + /// attribute later would look harmless and would take down the fleet. #[test] - fn the_shipped_storage_config_parses() { - // The migration section is operator-facing, so a typo in it would only surface on - // a node that had already shipped. Only `[storage]` is checked: the rest of - // `production.toml` does not currently deserialize as a `NodeConfig` (its - // `evm_network` is a bare string where an internally tagged enum is expected), - // which is a separate, pre-existing problem. - let raw = include_str!("../config/production.toml"); - let doc: toml::Value = toml::from_str(raw).expect("production.toml must be valid TOML"); - let storage = doc.get("storage").expect("a [storage] section").clone(); - let config: StorageConfig = storage.try_into().expect("[storage] must deserialize"); - - assert!(config.migration.enabled); - assert!(config.migration.dual_write_legacy); - assert_eq!(config.migration.shed_hold_hours, 72); - assert_eq!(config.migration.copier_throttle_mib_per_sec, 32); - assert_eq!(config.migration.copier_slack_mb, 2048); - // The release switches are absent from the file on purpose, so they come from the - // build rather than from whatever an operator's config last recorded. - let build = MigrationConfig::default(); - assert_eq!(config.migration.retire_legacy, build.retire_legacy); + fn a_config_file_from_the_previous_release_still_loads() { + let previous = r#" +[network] +port = 10000 + +[storage] +enabled = true +verify_on_read = true +db_size_gb = 32 +disk_reserve_mb = 500 + +[storage.migration] +shed_hold_hours = 72 +wave_hours = 24 +copier_throttle_mib_per_sec = 32 +copier_slack_mb = 2048 +retire_delay_hours = 4 + +[payment] +rewards_address = "0x0000000000000000000000000000000000000001" +"#; + let dir = tempfile::TempDir::new().expect("temp dir"); + let path = dir.path().join("config.toml"); + std::fs::write(&path, previous).expect("write the previous release's config"); + + // Through the loader a node actually uses, not a hand-picked table. The whole file + // has to parse, because that is what a node does with it on start. + let parsed = NodeConfig::from_file(&path) + .expect("a config file from the previous release must still load"); + + assert!(parsed.storage.enabled); + assert!(parsed.storage.verify_on_read); + assert_eq!( + parsed.storage.disk_reserve_mb, 500, + "the settings this build still uses must survive the ones it dropped" + ); } + use super::*; + use serial_test::serial; + #[test] fn test_default_config_has_cache_capacity() { let config = PaymentConfig::default(); diff --git a/src/devnet.rs b/src/devnet.rs index 5cf16b06..ececdbcd 100644 --- a/src/devnet.rs +++ b/src/devnet.rs @@ -602,7 +602,7 @@ impl Devnet { }; let storage = ChunkStore::new(storage_config) .await - .map_err(|e| DevnetError::Core(format!("Failed to create LMDB storage: {e}")))?; + .map_err(|e| DevnetError::Core(format!("Failed to create the chunk store: {e}")))?; let evm_config = EvmVerifierConfig { network: config diff --git a/src/lib.rs b/src/lib.rs index 83d19fec..98819613 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -71,7 +71,7 @@ pub use event::{NodeEvent, NodeEventsChannel}; pub use node::{NodeBuilder, RunningNode}; pub use payment::{PaymentStatus, PaymentVerifier, PaymentVerifierConfig}; pub use replication::{config::ReplicationConfig, ReplicationEngine}; -pub use storage::{AntProtocol, ChunkStore, ChunkStoreConfig, LmdbStorage, LmdbStorageConfig}; +pub use storage::{AntProtocol, ChunkStore, ChunkStoreConfig}; /// Re-exports from `saorsa-core` so downstream crates (e.g. `ant-client`) /// can depend on `ant-node` alone without a direct `saorsa-core` dependency. diff --git a/src/node.rs b/src/node.rs index aa5d1fe7..c8193793 100644 --- a/src/node.rs +++ b/src/node.rs @@ -39,14 +39,6 @@ use tokio_util::task::TaskTracker; #[cfg(unix)] use tokio::signal::unix::{signal, SignalKind}; -/// How long shutdown waits for the storage migration to reach a stopping point. -/// -/// Generous, because interrupting a copy mid-chunk costs nothing (every step is -/// idempotent and re-derived at the next start) but interrupting the drain that precedes -/// removing the legacy store is worth avoiding. Bounded, because a step that will not -/// finish must not hold the process open. -const MIGRATION_SHUTDOWN_GRACE: std::time::Duration = std::time::Duration::from_secs(30); - /// How long shutdown waits for in-flight request handlers to finish. /// /// Short, because these are single request/response exchanges and the peer will retry. @@ -114,10 +106,11 @@ impl NodeBuilder { // Ensure root directory exists std::fs::create_dir_all(&self.config.root_dir)?; - // One release-level decision, applied before anything can audit: while the fleet - // moves off the legacy chunk store, a peer is not penalised for failing to hold a - // chunk it was supposed to be holding. It is still penalised for failing a - // commitment-bound audit. Audits of both kinds run and record throughout. + // One release-level decision, applied before anything can audit. It stays suspended + // here: this release deletes the old chunk store, and a node that was away while the + // migration ran arrives holding one it cannot read, so restoring the accusation now + // would slash it in the release that stranded it. The release after this one restores + // it. The commitment-bound audit has penalised throughout and still does. crate::replication::config::apply_close_group_storage_penalty_policy(); // Create shutdown token @@ -160,6 +153,35 @@ impl NodeBuilder { (None, None) }; + // Only now, and only if a store was actually opened. Clearing up after the storage + // migration deletes directories the previous release had finished with, and + // "finished with" means their chunks are in the file store — which is only true if + // the file store is there. Doing this earlier put the deletion in front of a + // constructor that can still fail on an unreadable layout, a directory it cannot + // create, or a lock another process has not let go of, and a node that lost both + // stores that way had nothing to go back to. + // + // A node with storage switched off never builds one at all, so it never establishes + // that anything was copied anywhere, and it does not delete. It also has no use for + // the disk it would recover. Leaving the directory costs space on a node that is not + // storing anything anyway, and keeps its contents recoverable by the release that + // can read them. + // + // Waiting costs nothing in what this node reports. Its user agent was fixed when the + // transport was built a moment ago, but everything removed here is a leftover the + // signal already reads as finished with — carrying the mark or being empty is both + // what makes it removable and what makes it harmless — so the node announces `files` + // whether the deletion has finished, is still running, or has not started. + if ant_protocol.is_some() { + crate::storage::legacy_artifacts::clean_up(&self.config.root_dir); + } else { + info!( + "Chunk storage is disabled, so anything the storage migration left behind is \ + being left where it is: nothing here can establish that its chunks were \ + copied anywhere." + ); + } + let p2p_arc = Arc::new(p2p_node); // Wire the P2PNode handle into AntProtocol so payment proofs can query @@ -168,7 +190,7 @@ impl NodeBuilder { protocol.attach_p2p_node(Arc::clone(&p2p_arc)); } - let (replication_engine, migration_task) = match (&ant_protocol, fresh_write_rx) { + let replication_engine = match (&ant_protocol, fresh_write_rx) { (Some(protocol), Some(fresh_rx)) => { Self::build_replication_engine( protocol, @@ -181,7 +203,7 @@ impl NodeBuilder { ) .await? } - _ => (None, None), + _ => None, }; let node = RunningNode { @@ -194,7 +216,6 @@ impl NodeBuilder { ant_protocol, replication_engine, protocol_task: None, - migration_task, protocol_children: TaskTracker::new(), upgrade_exit_code: Arc::new(AtomicI32::new(-1)), }; @@ -202,18 +223,13 @@ impl NodeBuilder { Ok(node) } - /// Start the replication engine and, if this node still has one, the migration off - /// the legacy chunk store. - /// - /// The two are built together because the migration cannot run without the engine: - /// it needs the commitment state, which holds the veto on deleting the old store, - /// and the live routing view that says which chunks this node must never give up. + /// Start the replication engine. /// /// # Errors /// - /// Returns an error only when the engine fails to start on a node that has a legacy - /// store to migrate. On a node with nothing to migrate an engine failure is logged - /// and the node runs without one, as it always has. + /// Never, currently: an engine that fails to start is logged and the node runs without + /// one, as it always has. The signature keeps its `Result` because the caller's does, + /// and because the migration release did have a case that had to refuse. async fn build_replication_engine( protocol: &Arc, repl_config: ReplicationConfig, @@ -222,7 +238,7 @@ impl NodeBuilder { root_dir: &Path, fresh_rx: UnboundedReceiver, shutdown: &CancellationToken, - ) -> Result<(Option, Option>)> { + ) -> Result> { let engine = match ReplicationEngine::new( repl_config, Arc::clone(p2p), @@ -237,22 +253,8 @@ impl NodeBuilder { { Ok(engine) => engine, Err(e) => { - // A node that still has a legacy chunk store depends on this engine for - // the commitment state, the routing view and the possession challenges - // the migration cannot proceed without. Carrying on would leave it - // serving from both stores forever, never reclaiming its disk, which is - // the condition this release exists to end. Refuse to start instead of - // running in it indefinitely. - if protocol.storage().has_legacy() { - return Err(Error::Startup(format!( - "This node has a legacy chunk store to migrate but the \ - replication engine did not start: {e}. Without it the \ - migration cannot run and the disk is never reclaimed. \ - Fix the cause rather than running on." - ))); - } warn!("Failed to initialize replication engine: {e}"); - return Ok((None, None)); + return Ok(None); } }; @@ -274,10 +276,7 @@ impl NodeBuilder { .payment_verifier_arc() .attach_monetized_pin_sender(engine.monetized_pin_sender()); - let migration_task = - Self::spawn_storage_migration(protocol.storage(), p2p, &engine, shutdown.clone()); - - Ok((Some(engine), migration_task)) + Ok(Some(engine)) } /// Build the saorsa-core `NodeConfig` from our config. @@ -317,6 +316,26 @@ impl NodeBuilder { } } + // Say on the wire whether this node still has an old chunk store. It costs no new + // message and no new field: saorsa-core already sends a user agent with every signed + // message and keeps each peer's, so this is a different value in a string that was + // already there. It is the only thing that tells us anything at all about the nodes we + // do not run and have no logs from. It cannot establish that the fleet has finished: + // a node sees only the peers it is connected to, and each answers as of its own last + // start, so the most this shows is that some peer reported an old store when it last + // started. It can never show that no node has one. + // + // Read from the filesystem here rather than from the store, because the store is + // built later and a node with storage switched off never builds one at all, while + // the directory on its disk is just as real either way. + // + // Fixed for the life of the process: saorsa-core copies the string when it builds + // the transport. A node that finishes migrating goes on saying `legacy` until it + // restarts, which overstates how much is left rather than understating it, and is + // the direction a release gate should err in. + let signal = crate::storage::migration_signal::MigrationSignal::from_disk(&config.root_dir); + core_config.custom_user_agent = Some(crate::storage::migration_signal::user_agent(signal)); + // Persist close group peers + trust scores across restarts. // Default to root_dir (alongside node_identity.key) when not explicitly set. core_config.close_group_cache_dir = Some( @@ -448,58 +467,23 @@ impl NodeBuilder { monitor } - - /// Start moving this node off the legacy LMDB chunk store, if it still has one. - /// - /// Started after the replication engine rather than with the store, because the - /// copier needs two things only the engine has: the commitment state, which owns the - /// retention veto on deleting the old store, and live routing, which is how the node - /// knows which chunks it is among the closest to and therefore must never give up. - fn spawn_storage_migration( - store: Arc, - p2p: &Arc, - engine: &ReplicationEngine, - shutdown: CancellationToken, - ) -> Option> { - if !crate::storage::migration::should_migrate(&store) { - return None; - } - let context = crate::storage::migration::MigrationContext { - p2p: Some(Arc::clone(p2p)), - self_id: Some(*p2p.peer_id()), - self_xor: crate::client::peer_id_to_xor_name(&p2p.peer_id().to_string()), - commitment: Some(Arc::clone(engine.commitment_state())), - replication: Some(Arc::clone(engine.config())), - sync_state: Some(Arc::clone(engine.sync_state())), - audit_challenge_coordinator: Some(Arc::clone(engine.audit_challenge_coordinator())), - peer_commitments: Some(Arc::clone(engine.last_commitment_by_peer())), - close_group_size: engine.config().close_group_size, - }; - Some(tokio::spawn(async move { - crate::storage::migration::run(store, context, shutdown).await; - })) - } - /// Build the ANT protocol handler from config. /// - /// Initializes LMDB storage, payment verifier, and quote generator. + /// Initializes the chunk store, payment verifier, and quote generator. /// Wires ML-DSA-65 signing from the node's identity into the quote generator. async fn build_ant_protocol( config: &NodeConfig, identity: &NodeIdentity, close_group_size: usize, ) -> Result { - // Create LMDB storage let storage_config = ChunkStoreConfig { root_dir: config.root_dir.clone(), verify_on_read: config.storage.verify_on_read, - max_map_size: config.storage.db_size_gb.saturating_mul(1024 * 1024 * 1024), disk_reserve: config.storage.disk_reserve_mb.saturating_mul(MIB), - migration: config.storage.migration.clone(), }; let storage = ChunkStore::new(storage_config) .await - .map_err(|e| Error::Startup(format!("Failed to create LMDB storage: {e}")))?; + .map_err(|e| Error::Startup(format!("Failed to create the chunk store: {e}")))?; // Parse rewards address (required — node must know where to receive payments) let rewards_address = match config.payment.rewards_address { @@ -565,17 +549,12 @@ pub struct RunningNode { replication_engine: Option, /// Protocol message routing background task. protocol_task: Option>, - /// The task moving this node off the legacy chunk store, if it has one. - /// - /// Awaited before the replication engine and the P2P layer are torn down, because it - /// holds handles to both and is in the middle of reading and writing the chunk store. - migration_task: Option>, /// The per-message handler tasks the protocol loop spawns. /// /// Tracked rather than detached so shutdown can stop accepting work and then wait for /// what is already in flight. Aborting only the loop leaves its children running, and - /// a chunk read that outlives the loop keeps the legacy store busy exactly while the - /// migration is trying to drain it. + /// a chunk read that outlives the loop keeps working against a store the shutdown is + /// about to tear down. protocol_children: TaskTracker, /// Exit code requested by a successful upgrade (-1 = no upgrade exit pending). upgrade_exit_code: Arc, @@ -655,6 +634,22 @@ impl RunningNode { info!("Replication engine started"); } + // Say where this node is with the move off the old chunk store, and what it can see + // of its neighbours. The release that deletes that store may only go out once the + // fleet has moved, and no calendar establishes that: our own logs cover the nodes we + // run, and this is the only view we get of the ones we do not. + { + // Weak on purpose: see `report_until_shutdown`. A reporter that kept the node + // alive would keep its port bound after the node was dropped. + let p2p = Arc::downgrade(&self.p2p_node); + let root_dir = self.config.root_dir.clone(); + let shutdown = self.shutdown.clone(); + tokio::spawn(async move { + crate::storage::migration_signal::report_until_shutdown(p2p, root_dir, shutdown) + .await; + }); + } + // Start upgrade monitor if enabled if let Some(monitor) = self.upgrade_monitor.take() { let events_tx = self.events_tx.clone(); @@ -800,34 +795,26 @@ impl RunningNode { }); } - // A node that still has a legacy chunk store and no task moving it off one is the - // failure this cannot be allowed to have silently: the store opens, serves the - // union of both, and never frees a byte. It happened once, during a rebase that - // dropped the spawn, and nothing noticed because a node without a legacy store - // starts no migration and every test built the store directly. Say so loudly. - if let Some(ref protocol) = self.ant_protocol { - if protocol.storage().has_legacy() && self.migration_task.is_none() { - error!( - migration_event = "not_started", - "This node still has a legacy chunk store but nothing is migrating it. \ - Its disk will never be reclaimed. This is a wiring fault, not a \ - configuration one: report it rather than working around it." - ); - } - } - info!("Node running, waiting for shutdown signal"); - // Run the main event loop with signal handling + // The main event loop, with signal handling. Everything above this starts + // something; this is where the node waits. self.run_event_loop().await?; - // Protocol routing stops FIRST, loop and children both. The migration's last step - // drains the legacy store's in-flight reads, and inbound protocol traffic keeps - // starting new ones, so waiting on the migration while still serving requests can - // keep that drain from ever completing and hang shutdown. Aborting the accept loop - // alone would not do it: the requests already in flight run in their own tasks. + // Protocol routing stops first, loop and children both. The routing loop waits on + // `events.recv()` and has no cancellation branch of its own, and it holds an `Arc` + // on the P2P node that keeps the sender it is waiting on alive, so nothing else + // here will ever wake it. Left running it holds the chunk store and its + // single-process lock open after the node has returned. Aborting the accept loop + // alone is not enough either: the requests already in flight run in their own + // tasks, which is what the drain below is for. if let Some(handle) = self.protocol_task.take() { handle.abort(); + // Awaited, not just asked to stop. `abort` schedules cancellation; it does not + // establish that the task is gone, and what matters here is that it has + // dropped its `Arc` on the protocol and with it the store's single-process + // lock before this function returns. The join resolves as cancelled. + let _ = handle.await; } // Cancelled first, so anything still queued behind the concurrency permits gives // up rather than starting fresh storage work, then given a moment to finish what @@ -846,30 +833,6 @@ impl RunningNode { ); } - // Then the migration, awaited rather than aborted: it is mid-way through reading - // and writing the chunk store, and it holds the commitment state and the routing - // handle that the shutdown below is about to invalidate. It watches the same - // cancellation token, so this returns as soon as its current step does. Bounded, - // because a step that will not finish must not hold the process open. - if let Some(mut handle) = self.migration_task.take() { - // Awaited by reference, so a timeout leaves the handle here to abort rather - // than dropping it and letting the task run on detached through the engine and - // P2P teardown it depends on. - match tokio::time::timeout(MIGRATION_SHUTDOWN_GRACE, &mut handle).await { - Ok(Ok(())) => {} - Ok(Err(e)) => warn!("Storage migration task did not stop cleanly: {e}"), - Err(_) => { - warn!( - "Storage migration did not stop within {}s; stopping it. \ - Everything it does is idempotent and re-derived at the next start.", - MIGRATION_SHUTDOWN_GRACE.as_secs() - ); - handle.abort(); - let _ = handle.await; - } - } - } - // Shutdown replication engine before P2P so background tasks don't // use a dead P2P layer, and Arc references are released. if let Some(ref mut engine) = self.replication_engine { @@ -1101,6 +1064,8 @@ mod tests { use rand::Rng; use tempfile::TempDir; + use crate::storage::migration_signal::LEGACY_ENV_DIR; + /// The e2e port range, so a test bind never lands on a production or dev instance. const TEST_PORT_RANGE: std::ops::Range = 20000..60000; @@ -1110,79 +1075,6 @@ mod tests { /// A well-formed address that receives nothing; no chain is contacted in these tests. const TEST_REWARDS_ADDRESS: &str = "0x0000000000000000000000000000000000000001"; - /// A node with a legacy chunk store must get a migration task; one without must not. - /// - /// The spawn helper is tested directly because its *absence* is the failure mode that - /// already happened here: a rebase dropped the call, the store still opened and still - /// served, and no test could tell the difference. - #[tokio::test] - async fn a_legacy_store_gets_a_migration_task_and_a_fresh_node_does_not() { - let dir = TempDir::new().expect("temp dir"); - let root = dir.path().join("node"); - std::fs::create_dir_all(&root).expect("mkdir"); - - // Fresh node: nothing to migrate, so no task. - let fresh = Arc::new( - crate::storage::ChunkStore::new(crate::storage::ChunkStoreConfig { - root_dir: root.clone(), - ..crate::storage::ChunkStoreConfig::test_default() - }) - .await - .expect("open fresh"), - ); - assert!(!fresh.has_legacy()); - assert!( - !crate::storage::migration::should_migrate(&fresh), - "a node with no legacy store has nothing to migrate" - ); - drop(fresh); - - // Seed a legacy store, then reopen: now there is something to migrate. - { - let lmdb = crate::storage::LmdbStorage::new(crate::storage::LmdbStorageConfig { - root_dir: root.clone(), - verify_on_read: true, - max_map_size: 0, - disk_reserve: 0, - }) - .await - .expect("open legacy"); - let content = b"a chunk from before the migration"; - let addr = crate::client::compute_address(content); - lmdb.put(&addr, content).await.expect("put"); - lmdb.wait_idle().await; - } - let upgrading = Arc::new( - crate::storage::ChunkStore::new(crate::storage::ChunkStoreConfig { - root_dir: root.clone(), - ..crate::storage::ChunkStoreConfig::test_default() - }) - .await - .expect("open upgrading"), - ); - assert!(upgrading.has_legacy()); - assert!( - crate::storage::migration::should_migrate(&upgrading), - "a node with a legacy store must be migrated, or its disk is never reclaimed" - ); - } - - /// Seed a legacy LMDB store under `root` with one chunk, then close it. - async fn seed_legacy_store(root: &std::path::Path) { - let lmdb = crate::storage::LmdbStorage::new(crate::storage::LmdbStorageConfig { - root_dir: root.to_path_buf(), - verify_on_read: true, - max_map_size: 0, - disk_reserve: 0, - }) - .await - .expect("open legacy"); - let content = b"a chunk written before the migration"; - let addr = crate::client::compute_address(content); - lmdb.put(&addr, content).await.expect("put"); - lmdb.wait_idle().await; - } - /// A node config that builds without touching a chain or a real network. fn local_node_config(root: &std::path::Path, port: u16) -> NodeConfig { NodeConfig { @@ -1198,22 +1090,13 @@ mod tests { } } - /// A real, fully built node with a legacy store is actually migrating it. - /// - /// This goes through `build()` rather than calling the spawn helper, because the - /// failure that already happened here was the *call site* going missing, not the - /// helper being wrong. A test of the helper alone stays green through exactly that - /// bug. Deleting the spawn from `build()` must turn this red. + /// A node builds on a root with nothing left over from the old store. #[tokio::test] - async fn a_built_node_with_a_legacy_store_is_migrating_it() { + async fn a_node_builds_on_a_clean_root() { let dir = TempDir::new().expect("temp dir"); let root = dir.path().join("node"); std::fs::create_dir_all(&root).expect("mkdir"); - seed_legacy_store(&root).await; - // Ports are picked at random from the test range and a freshly released one can - // still be held for a moment, so a bind failure is retried rather than reported - // as a wiring fault. let mut built = None; let mut last_err = String::new(); for _ in 0..BIND_ATTEMPTS { @@ -1236,58 +1119,149 @@ mod tests { panic!("could not build a node after {BIND_ATTEMPTS} attempts: {last_err}"); }; - let storage_has_legacy = node - .ant_protocol - .as_ref() - .is_some_and(|p| p.storage().has_legacy()); - assert!( - storage_has_legacy, - "the node must have opened the legacy store this test seeded" - ); + node.shutdown.cancel(); + } + + /// A node with chunks in a store this build cannot read STARTS, however it is + /// configured, and does not lose them. + /// + /// This asserted the opposite until the release that shipped it was reconsidered twice. + /// First it refused to start, which serves nothing at all and cannot be recovered by + /// hand, because nothing can hold a node on an older build. Then it deleted whatever it + /// found, which is data loss for exactly the node that most needs the data: the upgrade + /// monitor picks the newest eligible release rather than the next one, so a node that was + /// offline through the previous release arrives here with everything it owns in that + /// directory and nothing in the file store. + /// + /// Both configurations, because they are different code paths: a node with + /// `storage.enabled = false` never builds a store, and the old environment is on its disk + /// just the same. + #[tokio::test] + async fn a_node_with_an_unmigrated_store_starts_and_keeps_it() { + for storage_enabled in [true, false] { + let dir = TempDir::new().expect("temp dir"); + let root = dir.path().join("node"); + let env = root.join(LEGACY_ENV_DIR); + std::fs::create_dir_all(&env).expect("mkdir"); + std::fs::write(env.join("data.mdb"), b"chunks that were never copied out") + .expect("seed"); + + let port = rand::thread_rng().gen_range(TEST_PORT_RANGE); + let mut config = local_node_config(&root, port); + config.storage.enabled = storage_enabled; + + let node = NodeBuilder::new(config).build().await.unwrap_or_else(|e| { + panic!( + "a node with an unmigrated store must start (storage.enabled = \ + {storage_enabled}): {e}" + ) + }); + settle(); + assert!( + env.join("data.mdb").exists(), + "chunks that were never copied out were deleted (storage.enabled = \ + {storage_enabled})" + ); + node.shutdown.cancel(); + } + } + + /// A node with storage switched off deletes nothing, even a leftover marked finished. + /// + /// It never builds a store, so nothing about this node establishes that those chunks were + /// copied anywhere: the mark is a claim made by a previous release about a file store + /// this process has not opened and will not open. It also has no use for the disk. So the + /// directory stays and stays recoverable. + #[tokio::test] + async fn a_node_with_storage_disabled_deletes_nothing() { + let dir = TempDir::new().expect("temp dir"); + let root = dir.path().join("node"); + let env = root.join(LEGACY_ENV_DIR); + std::fs::create_dir_all(&env).expect("mkdir"); + std::fs::write(env.join("data.mdb"), b"already copied out").expect("seed"); + std::fs::write(env.join("RETIRED"), b"retired").expect("mark"); + + let port = rand::thread_rng().gen_range(TEST_PORT_RANGE); + let mut config = local_node_config(&root, port); + config.storage.enabled = false; + let node = NodeBuilder::new(config).build().await.expect("must start"); + settle(); assert!( - node.migration_task.is_some(), - "a node holding a legacy chunk store came up with nothing migrating it, so \ - its disk would never be reclaimed" + env.join("data.mdb").exists(), + "a node that never opened a store deleted one on the strength of a mark it \ + could not check" ); + node.shutdown.cancel(); + } + + /// A store the migration finished with is removed, once the file store has opened. + #[tokio::test] + async fn a_finished_store_is_removed_once_the_replacement_opens() { + let dir = TempDir::new().expect("temp dir"); + let root = dir.path().join("node"); + let env = root.join(LEGACY_ENV_DIR); + std::fs::create_dir_all(&env).expect("mkdir"); + std::fs::write(env.join("data.mdb"), b"already copied out").expect("seed"); + std::fs::write(env.join("RETIRED"), b"retired").expect("mark"); + let port = rand::thread_rng().gen_range(TEST_PORT_RANGE); + let node = NodeBuilder::new(local_node_config(&root, port)) + .build() + .await + .expect("a node with a finished leftover must start"); + assert!(wait_gone(&env)); node.shutdown.cancel(); - if let Some(handle) = node.migration_task { - handle.abort(); - } } - /// A node with nothing to migrate does not start a driver for it. + /// Nothing is deleted until the store that replaced it has actually opened. + /// + /// "Finished with" means the chunks are in the file store, which is only true if the file + /// store opens. An earlier version deleted first and let the constructor fail behind it, + /// on an unreadable layout or a directory it could not create, and a node that lost both + /// stores that way had nothing left to go back to. #[tokio::test] - async fn a_built_node_without_a_legacy_store_starts_no_migration() { + async fn a_finished_store_survives_a_file_store_that_will_not_open() { let dir = TempDir::new().expect("temp dir"); let root = dir.path().join("node"); - std::fs::create_dir_all(&root).expect("mkdir"); + let env = root.join(LEGACY_ENV_DIR); + std::fs::create_dir_all(&env).expect("mkdir"); + std::fs::write(env.join("data.mdb"), b"already copied out").expect("seed"); + std::fs::write(env.join("RETIRED"), b"retired").expect("mark"); - let mut built = None; - let mut last_err = String::new(); - for _ in 0..BIND_ATTEMPTS { - let port = rand::thread_rng().gen_range(TEST_PORT_RANGE); - match NodeBuilder::new(local_node_config(&root, port)) - .build() - .await - { - Ok(node) => { - built = Some(node); - break; - } - Err(e) => { - last_err = e.to_string(); - tokio::time::sleep(std::time::Duration::from_millis(250)).await; - } + // A file where the chunk directory has to be, so the store cannot create it. + std::fs::write(root.join("chunks"), b"not a directory").expect("block the store"); + + let port = rand::thread_rng().gen_range(TEST_PORT_RANGE); + let built = NodeBuilder::new(local_node_config(&root, port)) + .build() + .await; + assert!( + built.is_err(), + "the file store was supposed to fail to open" + ); + settle(); + assert!( + env.join("data.mdb").exists(), + "the old store was deleted before the one replacing it could open" + ); + } + + /// The removal runs on its own thread. + fn settle() { + std::thread::sleep(std::time::Duration::from_millis(200)); + } + + /// The removal runs on its own thread, so give it a moment. + fn wait_gone(path: &std::path::Path) -> bool { + for _ in 0..200 { + if !path.exists() { + return true; } + std::thread::sleep(std::time::Duration::from_millis(10)); } - let Some(node) = built else { - panic!("could not build a node after {BIND_ATTEMPTS} attempts: {last_err}"); - }; - - assert!(node.migration_task.is_none()); - node.shutdown.cancel(); + false } + use super::*; use crate::config::NODES_SUBDIR; diff --git a/src/payment/metrics.rs b/src/payment/metrics.rs index b59c19f5..fe36002b 100644 --- a/src/payment/metrics.rs +++ b/src/payment/metrics.rs @@ -37,7 +37,7 @@ impl QuotingMetricsTracker { /// /// This is the deletion-aware path and the SINGLE source of truth for the /// priced record count: the handler calls it at quote time with the live - /// LMDB entry count (`current_chunks()`), so any record removed from + /// live chunk count (`current_chunks()`), so any record removed from /// storage — by delete, prune, or otherwise — is reflected on the next /// quote with no per-delete bookkeeping to keep in sync. `record_store` /// remains only an optimistic between-quote hint; the resync overwrites it. diff --git a/src/payment/verifier.rs b/src/payment/verifier.rs index 455f2491..ba3f0bfb 100644 --- a/src/payment/verifier.rs +++ b/src/payment/verifier.rs @@ -623,7 +623,7 @@ pub struct PaymentVerifier { /// midpoint in the live DHT. `None` in unit tests that don't exercise /// live-DHT checks; production startup MUST call [`attach_p2p_node`]. p2p_node: RwLock>>, - /// LMDB storage handle, attached post-construction. Retained for + /// Chunk store handle, attached post-construction. Retained for /// store-backed verifier checks that need the authoritative on-disk record /// count without depending on a side counter that may drift from /// replication/repair/prune paths. NOTE: the ADR-0006 price floor does NOT diff --git a/src/replication/admission.rs b/src/replication/admission.rs index 445d5644..23669ffa 100644 --- a/src/replication/admission.rs +++ b/src/replication/admission.rs @@ -202,7 +202,7 @@ mod tests { // ----------------------------------------------------------------------- // AdmissionResult construction helpers for pure-logic tests // - // The full `admit_hints` function requires a live DHT + LMDB backend. + // The full `admit_hints` function requires a live DHT and chunk store. // For unit tests we directly exercise: // 1. Cross-set precedence logic // 2. Deduplication logic diff --git a/src/replication/audit.rs b/src/replication/audit.rs index 9b312bb3..867b8368 100644 --- a/src/replication/audit.rs +++ b/src/replication/audit.rs @@ -896,9 +896,7 @@ mod tests { let config = ChunkStoreConfig { root_dir: temp_dir.path().to_path_buf(), verify_on_read: false, - max_map_size: 0, disk_reserve: 0, - ..ChunkStoreConfig::test_default() }; let storage = ChunkStore::new(config).await.expect("create storage"); (storage, temp_dir) diff --git a/src/replication/commitment_state.rs b/src/replication/commitment_state.rs index 9daff439..76cf3e66 100644 --- a/src/replication/commitment_state.rs +++ b/src/replication/commitment_state.rs @@ -945,6 +945,51 @@ fn prune_slots(inner: &mut Inner, now: Instant) { #[cfg(test)] #[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] mod tests { + + /// Retiring keeps an in-window pin answerable; clearing repudiates it. + /// + /// This is why the commitment rotation no longer has a "storage is empty" branch. That + /// branch called `clear_all`, and it was reached from a key count that was wrong in the + /// same direction every time it was fixed: a node whose disk filled before it could copy + /// anything, then one whose file had been dropped from the index by a failed read, then a + /// files-only node that had published bytes but not yet indexed them. + /// + /// The cost of being wrong is what settles it, and it is what this measures. Clearing + /// repudiates a root a peer is still pinning, which answers `UnknownCommitment` and is + /// graded a confirmed failure. Retiring stops advertising and stays answerable until the + /// gossip TTL lapses. Both stop advertising; only one throws the answer away. + #[test] + fn retiring_keeps_a_pinned_root_answerable_and_clearing_does_not() { + let (pk, sk) = keypair(); + let pk_bytes = pk.to_bytes(); + let peer_id = *blake3::hash(&pk.to_bytes()).as_bytes(); + + let retired = ResponderCommitmentState::new(); + let c = BuiltCommitment::build(vec![(key(1), bh(1))], &peer_id, &sk, &pk_bytes).unwrap(); + let h = c.hash(); + retired.rotate(c); + retired.mark_gossiped(h); + retired.retire_current(); + assert!( + retired.current().is_none(), + "retiring must stop the node advertising the root" + ); + assert!( + retired.lookup_by_hash(&h).is_some(), + "but a peer still pinning it must get an answer, not a repudiation" + ); + + let cleared = ResponderCommitmentState::new(); + let c2 = BuiltCommitment::build(vec![(key(1), bh(1))], &peer_id, &sk, &pk_bytes).unwrap(); + let h2 = c2.hash(); + cleared.rotate(c2); + cleared.mark_gossiped(h2); + cleared.clear_all(); + assert!( + cleared.lookup_by_hash(&h2).is_none(), + "clearing throws the same pin away, which is the confirmed failure this avoids" + ); + } use super::*; use crate::replication::commitment::{commitment_hash, leaf_hash, verify_path}; use saorsa_pqc::api::sig::ml_dsa_65; diff --git a/src/replication/config.rs b/src/replication/config.rs index 488b8899..e51e72eb 100644 --- a/src/replication/config.rs +++ b/src/replication/config.rs @@ -230,7 +230,7 @@ pub const AUDIT_RESPONDER_TOP_ORIGINS: usize = 10; /// round-1 proofs from starving the light audits, and bounds concurrent /// multi-gigabyte hashing to this many at once. Two allows overlap without /// admitting many simultaneous full-subtree hashes; there is little benefit in -/// more concurrent large LMDB scans against one disk. +/// more concurrent large store scans against one disk. pub const MAX_CONCURRENT_SUBTREE_ROUND1: usize = 2; /// Per-peer concurrency cap for the heavy subtree-audit round 1. One in-flight @@ -293,7 +293,7 @@ pub const SUBTREE_ROUND1_WORK_BURST_BYTES: i64 = 8 * 1024 * 1024 * 1024; /// Floor charged against the round-1 work budget per leaf attempted, in bytes. /// /// The budget counts content bytes, which is the right unit for the hashing but -/// misses what a leaf costs before its size is known: an LMDB point lookup with +/// misses what a leaf costs before its size is known: a point lookup with /// its retries, and a `spawn_blocking` dispatch and join. Nothing bounds a /// chunk from below, so a commitment made of a million tiny records would run a /// full subtree of reads and task round-trips per audit while charging almost @@ -631,7 +631,7 @@ pub const MAX_VERIFICATION_KEYS_PER_CYCLE: usize = 8_192; /// /// Senders aggregate all keys for a peer into one request. Matching this limit /// to the cycle bound lets an honest round use one request per peer while still -/// bounding the LMDB work performed on the responder's serial replication +/// bounding the storage work performed on the responder's serial replication /// message path. Oversized requests are rejected as an empty, wire-compatible /// verification response. pub const MAX_INCOMING_VERIFICATION_KEYS: usize = MAX_VERIFICATION_KEYS_PER_CYCLE; @@ -693,10 +693,25 @@ pub(crate) const CAPACITY_BLOCKED_RETRY: Duration = /// Trust event weight for confirmed audit failures. pub const AUDIT_FAILURE_TRUST_WEIGHT: f64 = 5.0; -/// Whether this build penalises a peer for not holding a chunk it was supposed to hold. +/// Whether this build HOLDS OFF the penalty for not holding a chunk it was supposed to hold. /// -/// **`true` while the fleet moves off the legacy LMDB chunk store; back to `false` once it -/// has.** Flipping it is a one-line change in one release. +/// `true` suspends the penalty. The name says `SUSPEND`; read it that way, because a reader who +/// takes it as "does this build penalise" gets the answer backwards, which is how a comment +/// three lines down came to claim the opposite of what ships. +/// +/// **Still `true`, and deliberately not flipped by this release.** It was raised while the +/// fleet moved off the legacy LMDB chunk store, because a node that has to give up chunks +/// cannot stop its peers penalising it for that, so the peers had to stop first. +/// +/// Restoring it belongs in a release *after* this one, not in this one. The upgrade monitor +/// picks the newest eligible release rather than the next, so a node that was offline while +/// the migration ran arrives here having never migrated, holding a legacy store this build +/// cannot read. This release keeps that store rather than deleting it, which is the right +/// answer for its data, but the node cannot serve those chunks. Restoring the penalty in the +/// same release would slash exactly that node, for a state it had no chance to leave, in the +/// release that stranded it. Two changes, two releases: this one removes the old store, and +/// the next one restores the accusation once the fleet has been observed clean for long +/// enough to include the nodes that were away. /// /// Deliberately narrow. It covers exactly one accusation: "you did not have a chunk you /// were supposed to be holding". It does **not** cover the commitment-bound subtree audit, @@ -714,6 +729,19 @@ pub const AUDIT_FAILURE_TRUST_WEIGHT: f64 = 5.0; pub const RELEASE_SUSPEND_CLOSE_GROUP_STORAGE_PENALTY: bool = true; /// Environment override for [`RELEASE_SUSPEND_CLOSE_GROUP_STORAGE_PENALTY`], for a canary. +/// +/// Kept rather than removed with the rest of the bridge. This release does not flip the +/// constant, but the release after it does, and that is the moment the lever is most likely to +/// be needed. It suspends only the penalties this node hands out, so an emergency suspension +/// has to go to the fleet, not to the node being penalised. +/// +/// The one-way guard the previous release added is kept, and in this release it is ACTIVE: it +/// refuses to un-suspend while the release constant says to hold the penalty off, and this +/// release says exactly that. So the override can suspend and cannot un-suspend, and a host +/// setting it to `0` is told so and ignored — which is the point, because one host penalising +/// its close group for behaving as the release asked would slash all of them. It becomes inert +/// in the release that restores the penalty, and it stays there rather than being deleted so +/// that the meaning does not silently change if the constant ever goes back. pub const SUSPEND_CLOSE_GROUP_STORAGE_PENALTY_ENV: &str = "ANT_SUSPEND_UNHELD_CHUNK_PENALTY"; /// The live switch. @@ -736,7 +764,7 @@ pub fn apply_close_group_storage_penalty_policy() { apply_and_announce(RELEASE_SUSPEND_CLOSE_GROUP_STORAGE_PENALTY); return; }; - let suspended = match raw.trim().to_ascii_lowercase().as_str() { + let asked = match raw.trim().to_ascii_lowercase().as_str() { "1" | "true" | "yes" | "on" => true, "0" | "false" | "no" | "off" => false, other => { @@ -747,7 +775,22 @@ pub fn apply_close_group_storage_penalty_policy() { RELEASE_SUSPEND_CLOSE_GROUP_STORAGE_PENALTY } }; - apply_and_announce(suspended); + // Suspending is always allowed. Un-suspending is not, while the release says to hold + // off: this node would hand out full-weight trust failures to peers that are doing what + // the release asked of them, and they have no way to stop it or to see where it came + // from. + if !asked && RELEASE_SUSPEND_CLOSE_GROUP_STORAGE_PENALTY { + warn!( + "{SUSPEND_CLOSE_GROUP_STORAGE_PENALTY_ENV} asks this node to penalise peers for \ + not holding a close-group chunk, and this release holds that penalty off while \ + the fleet moves off the old chunk store. Ignoring it: a node giving chunks up \ + cannot stop a peer punishing it for doing so, so one host set this way would \ + slash its whole close group for behaving correctly." + ); + apply_and_announce(RELEASE_SUSPEND_CLOSE_GROUP_STORAGE_PENALTY); + return; + } + apply_and_announce(asked); } /// Set the switch and say so, once, where an operator will see it. @@ -1424,6 +1467,41 @@ mod tests { use super::*; use serial_test::serial; + /// The override can hold the penalty off. It cannot switch it back on. + /// + /// The direction is the whole point. A node giving chunks up during the migration cannot + /// stop a peer punishing it for doing so, so one host with this set the wrong way is not + /// a local choice about that host: it hands full-weight trust failures to every node in + /// its close group that is doing what the release asked. Set it right and it is a no-op + /// this release; set it wrong and it is a slashing incident nobody can trace. + #[test] + #[serial] + fn the_penalty_override_can_only_hold_the_penalty_off() { + // The switch is process-wide by design and this test is serialised, so nothing + // else is reading the variable or the switch while it runs. + std::env::set_var(SUSPEND_CLOSE_GROUP_STORAGE_PENALTY_ENV, "0"); + apply_close_group_storage_penalty_policy(); + assert_eq!( + close_group_storage_penalty_suspended(), + RELEASE_SUSPEND_CLOSE_GROUP_STORAGE_PENALTY, + "an operator switched the penalty back on and would slash their own close group" + ); + + std::env::set_var(SUSPEND_CLOSE_GROUP_STORAGE_PENALTY_ENV, "1"); + apply_close_group_storage_penalty_policy(); + assert!( + close_group_storage_penalty_suspended(), + "holding the penalty off is the safe direction and must still work" + ); + + std::env::remove_var(SUSPEND_CLOSE_GROUP_STORAGE_PENALTY_ENV); + apply_close_group_storage_penalty_policy(); + assert_eq!( + close_group_storage_penalty_suspended(), + RELEASE_SUSPEND_CLOSE_GROUP_STORAGE_PENALTY + ); + } + #[test] fn defaults_pass_validation() { let config = ReplicationConfig::default(); @@ -1457,6 +1535,28 @@ mod tests { /// One test rather than several, because the switch is process-wide: separate tests /// would race each other under the default parallel runner. + #[test] + #[serial] + fn this_release_still_holds_the_unheld_chunk_penalty_off() { + // Named on purpose, because the value matters and nothing else asserts it. This is + // the release that deletes the old chunk store, and the upgrade monitor picks the + // newest eligible release rather than the next one, so a node that was away while + // the migration ran arrives here having never migrated. It keeps its legacy store, + // which this build cannot read, so it cannot serve those chunks. Restoring the + // accusation here would slash that node in the release that stranded it. + // + // Restoring it is a one-line change in the release after this one, once the fleet + // has been observed clean for long enough to include the nodes that were away. + // Asked of the live switch after applying the policy rather than of the constant: + // clippy rejects an assertion on a constant, and what a node actually does is the + // better question. + apply_and_announce(RELEASE_SUSPEND_CLOSE_GROUP_STORAGE_PENALTY); + assert!( + close_group_storage_penalty_suspended(), + "this release must not restore the penalty; it can strand a node it then slashes" + ); + } + #[test] #[serial] fn the_unheld_chunk_penalty_switch_follows_the_release_it_is_compiled_into() { diff --git a/src/replication/mod.rs b/src/replication/mod.rs index 71124727..f43fb7da 100644 --- a/src/replication/mod.rs +++ b/src/replication/mod.rs @@ -992,7 +992,7 @@ const INBOUND_REPLICATION_SERIAL_QUEUE_CAPACITY: usize = 64; /// Maximum fresh-replication offers processed concurrently, away from the /// serial non-audit loop. /// -/// Fresh offers can perform an on-chain payment verification and a 4 MiB LMDB +/// Fresh offers can perform an on-chain payment verification and a 4 MiB /// write. Four workers keep that latency off the responder dispatch path while /// keeping concurrent EVM/storage pressure small and predictable. const FRESH_OFFER_WORKER_LIMIT: usize = 4; @@ -1126,7 +1126,7 @@ const FETCH_RESPONDER_MAX_OUTSTANDING_PER_PEER: u32 = 2; /// Maximum verification batches served concurrently. /// -/// LMDB point lookups are fast, but a batch can contain 8,192 of them. Two +/// Point lookups are fast, but a batch can contain 8,192 of them. Two /// workers isolate that synchronous work from message dispatch without turning /// large batches into an I/O fan-out throughput contest. const VERIFICATION_RESPONDER_WORKER_LIMIT: usize = 2; @@ -1484,13 +1484,13 @@ const BOOTSTRAP_DRAIN_CHECK_SECS: u64 = 5; /// observe the cancellation token and terminate before aborting it. /// /// Detached tasks are drained without a timeout because storage-capable work -/// may be awaiting a `spawn_blocking` LMDB operation, which continues running +/// may be awaiting a `spawn_blocking` storage operation, which continues running /// if its async waiter is dropped. const SHUTDOWN_TASK_DRAIN_TIMEOUT: Duration = Duration::from_secs(10); /// How often the responder rebuilds + rotates its storage commitment. /// -/// Each rebuild scans LMDB to compute leaf hashes; for ~10k keys this is +/// Each rebuild scans the store to compute leaf hashes; for ~10k keys this is /// sub-100ms (BLAKE3 + tree build). Retention is gossip-anchored, NOT /// rotation-anchored: the responder stays answerable for the current /// commitment plus every root it recently gossiped that is still in-window @@ -1700,7 +1700,7 @@ pub struct ReplicationEngine { identity: Arc, /// Responder-side commitment state (two-slot atomic rotation). /// - /// Periodically rebuilt from the live LMDB key set; gossiped on + /// Periodically rebuilt from the live key set; gossiped on /// outbound `NeighborSyncRequest`/`Response`; consulted by the /// commitment-bound audit handler. commitment_state: Arc, @@ -2283,14 +2283,14 @@ impl ReplicationEngine { /// /// This must be awaited before dropping the engine when the caller needs /// the `Arc` references held by background tasks to be - /// released (e.g. before reopening the same LMDB environment). + /// released (e.g. before reopening the same store). /// /// When this returns, no engine-spawned task still holds - /// `Arc` or `Arc`, and no LMDB blocking operation - /// (read or write, on either the chunk store or the paid-list + /// `Arc` or `Arc`, and no blocking storage operation + /// (read or write, against either the chunk store or the paid-list LMDB /// environment) is still running. Engine tasks race their work against /// the shutdown token; a dropped future may leave a `spawn_blocking` - /// LMDB transaction running detached, so this method additionally waits + /// operation running detached, so this method additionally waits /// for both storage layers to go quiescent before returning. pub async fn shutdown(&mut self) { self.shutdown.cancel(); @@ -2331,11 +2331,12 @@ impl ReplicationEngine { // All producers have stopped, so close and drain their detached work. // A started storage operation must run to completion: dropping an async // waiter does not cancel `spawn_blocking`, and would let shutdown return - // while an LMDB transaction still owns the environment. + // while a blocking storage operation is still running. // - // Deliberately unbounded: the LMDB contract requires every worker to - // release its `Arc` before the caller may reopen the - // environment, and a timeout here could return with one still held. + // Deliberately unbounded: every worker has to release its + // `Arc` before the caller may reopen the store, whose lock + // admits one process at a time, and a timeout here could return with one + // still held. // What makes that safe is that every detached task is now guaranteed to // finish — the pools above are closed, stale work is shed at dequeue, // and the one genuinely unbounded await (payment verification) races @@ -2344,7 +2345,7 @@ impl ReplicationEngine { self.detached_task_tracker.wait().await; // Every producer is gone, but a select! racing the shutdown token may - // have dropped a future while it awaited an LMDB `spawn_blocking` op + // have dropped a future while it awaited a storage `spawn_blocking` op // (fetch `storage.put`, prune `storage.delete` / // `paid_list.remove_batch`, verification `paid_list.insert`). The // detached blocking closure owns a cloned `Env`; wait for both @@ -2481,12 +2482,12 @@ impl ReplicationEngine { // so those waiters would drain only at the probe timeout // (roughly `queued / per-target-limit` probes deep) while // `detached_task_tracker.wait()` — deliberately unbounded - // for the LMDB contract — held shutdown open. + // for the storage contract — held shutdown open. // // Dropping this future mid-probe is safe and is the same // shape the neighbor-sync round uses: a parked coordinator // acquire releases its counted reference via - // `ReferenceGuard`, and a dropped LMDB `spawn_blocking` is + // `ReferenceGuard`, and a dropped storage `spawn_blocking` is // covered by the storage-quiescence wait in `shutdown`. tokio::select! { () = shutdown.cancelled() => {} @@ -3337,7 +3338,7 @@ impl ReplicationEngine { /// /// Phase 3 of the v12 storage-bound audit. Once per /// [`COMMITMENT_ROTATION_INTERVAL_SECS`], the responder reads the - /// current LMDB key set, builds a Merkle tree (for content-addressed + /// current key set, builds a Merkle tree (for content-addressed /// chunks `bytes_hash == key`, so no chunk re-read is needed), signs /// the root with the node's `MlDsaSecretKey`, and rotates the result /// into `commitment_state`. Old `previous` slot is dropped by the @@ -4310,7 +4311,7 @@ struct ReplicationMessageHandlerContext { /// The engine's shutdown token, for detached responder work. /// /// Workers on [`Self::detached_task_tracker`] race this around their - /// *network* phase only — never around an LMDB `spawn_blocking` await, + /// *network* phase only — never around a storage `spawn_blocking` await, /// where dropping the awaiter would detach a live transaction. This is /// what lets `shutdown()` keep its unbounded `tracker.wait()` and still /// terminate: the wait stays safe because it is now guaranteed finite. @@ -5487,7 +5488,7 @@ async fn handle_replication_message( /// is guaranteed to end. /// /// Deliberately NOT applied to `storage.put`: that awaits `spawn_blocking`, so -/// dropping its awaiter would detach a live LMDB transaction and break the +/// dropping its awaiter would detach a live storage operation and break the /// very contract the unbounded wait exists to uphold. async fn verify_payment_until_shutdown( payment_verifier: &Arc, @@ -5717,7 +5718,7 @@ async fn refuse_stranded_fresh_offers( /// /// This runs on the serial non-audit message loop, so it must stay cheap: every /// path here is a set insert, a permit try, or a small response send. The offer -/// itself — an on-chain payment verification and a multi-MiB LMDB write — always +/// itself — an on-chain payment verification and a multi-MiB write — always /// runs on a tracked worker task, never inline, because stalling this loop backs /// up the inbound queue and ultimately drops replication messages wholesale. /// @@ -5874,7 +5875,7 @@ async fn dispatch_fresh_offer( /// /// Split out so `dispatch_fresh_offer` stays a readable admission decision. /// A started handler is never cancelled: `storage.put()` awaits -/// `spawn_blocking`, and dropping that awaiter would detach the live LMDB +/// `spawn_blocking`, and dropping that awaiter would detach the live storage /// transaction. Shutdown responsiveness comes from the closed worker semaphore /// and from `handle_fresh_offer` racing the token around payment verification. /// @@ -6997,9 +6998,12 @@ fn request_is_stale(received_at: Instant, timeout: Duration) -> bool { enum FetchFault { /// The peer does not hold a chunk it was expected to hold. /// - /// This is the lane the release withholds, because a node part-way through moving - /// off the legacy store answers exactly this way about chunks it has legitimately - /// given up. + /// This is the lane the migration releases withhold, because a node part-way through + /// moving off the old store answers exactly this way about chunks it had legitimately + /// given up. It is STILL withheld here: this release deletes the old store, and a node + /// that was away while the migration ran arrives holding one it cannot read, so + /// accusing it in the release that stranded it would slash it for a state it had no + /// chance to leave. The release after this one restores it. UnheldChunk, /// The peer's own storage failed, or served bytes that no longer hash to their /// address. @@ -8970,7 +8974,7 @@ async fn execute_single_fetch( if let Err(e) = storage.put(&resp_key, &data).await { // The bytes arrived and passed the content-address // check, so the source did its job; the failure is - // entirely local (disk-full, or an LMDB error). Any + // entirely local (disk-full, or a storage error). Any // valid source must serve identical content, so trying // the next one cannot cure a local error — it only // re-downloads the same chunk into the same store. @@ -9941,7 +9945,7 @@ async fn write_retention_atomic(path: &Path, bytes: Vec) -> bool { } } -/// Read the current LMDB key set, build + sign a fresh +/// Read the current key set, build + sign a fresh /// `StorageCommitment`, and rotate it into `state` as the new `current`. /// The prior `current` is demoted to `previous`; the prior `previous` is /// dropped (per `ResponderCommitmentState::rotate`). @@ -9960,13 +9964,11 @@ async fn rebuild_and_rotate_commitment( p2p: &Arc, config: &Arc, ) -> Result<()> { - // Not `all_keys()`. While the node is bridging off the legacy store these are the - // same thing, but once it has settled on what it can hold this narrows to the - // file-backed set, which is what stops it claiming keys it is about to give up. It is - // also what lets `is_held` eventually go false for those keys, which is the gate on - // removing the legacy environment at all. + // Not `all_keys()`: that is every name the store holds, and what belongs in a commitment + // is only what this node is still responsible for. The sentence that used to be here was + // cut off mid-way and described the bridge, which no longer exists. let stored_keys = storage - .committable_keys() + .all_keys() .await .map_err(|e| Error::Storage(format!("commitment build: read keys: {e}")))?; @@ -9980,7 +9982,6 @@ async fn rebuild_and_rotate_commitment( // this filter the pruner's reprieve would keep re-committing stale keys // forever (the rebuild reads all_keys, so a retained-on-disk key would be // re-committed and re-gossiped every rotation — a permanent pin). - let storage_empty = stored_keys.is_empty(); let self_id = *p2p.peer_id(); let mut keys = Vec::with_capacity(stored_keys.len()); for k in stored_keys { @@ -9990,20 +9991,30 @@ async fn rebuild_and_rotate_commitment( } if keys.is_empty() { - if storage_empty { - // Storage is genuinely empty — there is nothing to answer for, so - // drop the previously advertised commitment immediately. Keeping it - // would leave remote auditors pinning a hash we can never satisfy - // again (the bytes are gone). - if state.retained_slot_count() > 0 { - debug!("Commitment rotation: storage empty, clearing retained slots"); - state.clear_all(); - } - storage.note_commitment_rebuilt(); - return Ok(()); - } - // Bytes are still on disk but no key is currently in range. We must NOT - // clear retention here: a peer may still be pinning a root we gossiped + // There used to be a second branch here that dropped every retained root outright + // when the node looked empty. It is gone, and the reason is worth keeping. + // + // "Empty" was decided from key counts, and every version of that test was wrong in + // the same direction. It read the committable set, which narrows to the file-backed + // keys once the migration settles, so a node whose disk filled before it could copy + // anything looked empty with a full legacy store beside it. Adding the raw file index + // still missed a file dropped from the index by a failed read while its legacy copy + // was being put back. Adding the legacy environment still missed a files-only node + // that had published bytes to disk but not yet indexed them, because a file is + // published before it is indexed. Each fix closed one window and left another. + // + // The asymmetry is what settles it. Clearing wrongly repudiates a root a peer is + // pinning, and `UnknownCommitment` is a confirmed failure on the commitment-bound + // lane, which is enforced in every release and is not the lane the migration holds + // off — so a node that still holds the bytes is slashed for holding them. Retiring + // wrongly costs a root that stops being advertised now and ages out by its gossip TTL + // instead of vanishing now. Both set `has_current = false`; they differ only in + // whether the node goes on being answerable in the meantime. A genuinely empty node + // cannot answer either way, so retiring costs it nothing it had. + // + // So there is one branch, and no emptiness question to get wrong. + // + // A peer may still be pinning a root we gossiped // moments ago and could demand its bytes in a round-2 challenge, which // we can still answer (the bytes are present). But we must STOP // advertising the stale commitment: retire it so `current()` returns @@ -10019,7 +10030,6 @@ async fn rebuild_and_rotate_commitment( (stays answerable until its gossip TTL lapses, bytes still on disk)" ); state.retire_current(); - storage.note_commitment_rebuilt(); return Ok(()); } @@ -10086,9 +10096,6 @@ async fn rebuild_and_rotate_commitment( // committed key set is frozen here for many rotations. Without this, // the no-op guard would pin a stale slot — and its key — forever. state.age_out(); - // The advertised commitment already equals the committable set, which is - // exactly what the retirement gate is counting. - storage.note_commitment_rebuilt(); return Ok(()); } } @@ -10111,10 +10118,6 @@ async fn rebuild_and_rotate_commitment( let key_count = built.commitment().key_count; state.rotate(built); info!("Storage commitment rotated: hash={hash} key_count={key_count}"); - // Counted only on the paths where the advertised commitment now genuinely reflects - // the committable set, never merely on having read it. The retirement gate is what - // consumes this, and it authorises deleting the legacy store. - storage.note_commitment_rebuilt(); Ok(()) } @@ -10155,15 +10158,14 @@ mod tests { /// to a response is what the classification above rests on. /// /// A key the peer does not hold reads as `Ok(None)`. A read that fails, whether from - /// an I/O fault or a failed integrity check, reads as `Err`. Nothing in the migration - /// turns the first into the second. + /// an I/O fault or a failed integrity check, reads as `Err`. Nothing turns the first + /// into the second. #[tokio::test] async fn a_missing_key_reads_as_a_plain_miss_and_a_failed_read_as_a_fault() { let dir = tempfile::tempdir().expect("temp dir"); - let storage = crate::storage::LmdbStorage::new(crate::storage::LmdbStorageConfig { + let storage = crate::storage::ChunkStore::new(crate::storage::ChunkStoreConfig { root_dir: dir.path().to_path_buf(), verify_on_read: true, - max_map_size: 0, disk_reserve: 0, }) .await diff --git a/src/replication/pruning.rs b/src/replication/pruning.rs index 9d213126..aa503757 100644 --- a/src/replication/pruning.rs +++ b/src/replication/pruning.rs @@ -1241,53 +1241,6 @@ async fn collect_record_prune_proofs( present_by_key } -/// Prove that other nodes actually hold `keys`, by cryptographic challenge. -/// -/// Exposed for the storage migration, which has to answer the same question the pruner -/// answers before it deletes: is this chunk somewhere else? It deliberately reuses this -/// path rather than the cheaper `VerificationRequest`, because that one carries a -/// self-reported `present: bool` and a node that has silently lost a chunk will still say -/// yes. Here the peer has to return `compute_audit_digest(nonce, peer, key, bytes)` over a -/// nonce it has never seen, which it cannot do without the bytes. -/// -/// Returns, per key, the set of peers that proved possession. The caller decides how many -/// are enough; [`prune_proofs_needed`] is the rule the pruner uses. -pub(crate) async fn prove_peers_hold_records( - keys_by_peer: &HashMap>, - local_stored_key_count: usize, - storage: &Arc, - p2p_node: &Arc, - config: &ReplicationConfig, - sync_state: &Arc>, - audit_challenge_coordinator: &Arc, -) -> HashMap> { - if keys_by_peer.is_empty() { - return HashMap::new(); - } - let candidates: Vec = { - let mut by_key: HashMap> = HashMap::new(); - for (peer, keys) in keys_by_peer { - for key in keys { - by_key.entry(*key).or_default().push(*peer); - } - } - by_key - .into_iter() - .map(|(key, target_peers)| RecordPruneCandidate { key, target_peers }) - .collect() - }; - collect_record_prune_proofs( - &candidates, - local_stored_key_count, - storage, - p2p_node, - config, - sync_state, - audit_challenge_coordinator, - ) - .await -} - async fn revalidated_fast_prune_keys( candidates: &[FastPruneCandidate], ctx: &PrunePassContext<'_>, diff --git a/src/replication/storage_commitment_audit.rs b/src/replication/storage_commitment_audit.rs index f99a4e70..12f86853 100644 --- a/src/replication/storage_commitment_audit.rs +++ b/src/replication/storage_commitment_audit.rs @@ -1250,7 +1250,7 @@ pub async fn handle_subtree_challenge( pub struct Round1Work { /// What to send back. pub response: SubtreeAuditResponse, - /// Chunk content read from LMDB and hashed BEFORE this response was + /// Chunk content read from the store and hashed BEFORE this response was /// produced. /// /// Counted on the rejecting paths too, which is the point. A subtree is read @@ -1363,7 +1363,7 @@ async fn subtree_challenge_response( let mut leaves = Vec::with_capacity(plan.leaf_keys.len()); for key in &plan.leaf_keys { // Charge the fixed cost of ATTEMPTING a leaf before the read, because - // it is owed whether or not the read succeeds: the LMDB lookup and its + // it is owed whether or not the read succeeds: the lookup and its // retries, and the blocking-task round trip below. Charging only // content bytes left both a failing leaf and a tiny one nearly free, // and nothing bounds a chunk from below, so a commitment of a million @@ -1603,7 +1603,7 @@ pub async fn handle_subtree_slice_challenge( }; // Coalesce openings by key, preserving first-seen order and deduplicating - // block indices per key, so each committed chunk is read from LMDB and hashed + // block indices per key, so each committed chunk is read from the store and hashed // at most once even when the auditor opens several of its blocks (the normal // random + final pair, or a forged duplicate). Without this a ten-opening // request could re-read and re-hash the same chunk ten times. @@ -1758,7 +1758,7 @@ async fn serve_committed_key_openings( } // Persistent transient read error after retries → do NOT brand the peer a // deleter. Reject `Transient`; the auditor routes it to the timeout lane - // so a flaky LMDB read never manufactures a confirmed possession failure + // so a flaky read never manufactures a confirmed possession failure // on an honest holder (which also gains no credit). Err(e) => { warn!( diff --git a/src/storage/chunk_store.rs b/src/storage/chunk_store.rs index fd4c7435..468ffd77 100644 --- a/src/storage/chunk_store.rs +++ b/src/storage/chunk_store.rs @@ -1,111 +1,203 @@ -//! The node's chunk store: a file store, plus the legacy LMDB environment for as long -//! as one still exists on disk. +//! One immutable file per chunk, content-addressed, with the filesystem as the +//! only authority. //! -//! Every caller in the node talks to this type and sees **one** key set. That is the -//! detail that keeps quoting, commitments, hints, audits and pruning coherent while a -//! chunk moves from LMDB to a file: the backing changes, the logical key set does not. +//! ```text +//! {root}/chunks/ store root +//! {root}/chunks/layout.json versioned layout marker +//! {root}/chunks/.lock advisory single-process guard +//! {root}/chunks//<64-hex> xy = the LAST two hex characters of the address +//! {root}/chunks//.tmp.. an in-flight write, in the destination directory +//! ``` //! -//! There is one deliberate asymmetry, and it is the whole safety argument of the -//! migration. Serving reads the **union**, so the node answers for everything it ever -//! committed to. The commitment builder reads only the **file-backed** set once the node -//! has settled on what it will keep, so the node stops claiming keys it is about to give -//! up. Between those two, a node is at worst over-honest: it serves more than it claims. +//! # Why the *last* two hex characters +//! +//! A node holds keys for which it is among the [`CLOSE_GROUP_SIZE`] closest, so its +//! holdings share roughly `log2(N / CLOSE_GROUP_SIZE)` leading bits with its own node +//! ID, and that shared prefix grows as the network grows. Sharding on a prefix therefore +//! does not degrade, it collapses: at ~800 nodes a two-hex prefix already resolves to +//! about two distinct directories, and past a million nodes even a four-hex prefix +//! resolves to one. Close-group membership constrains the leading bits and places no +//! constraint at all on the trailing ones, and the address is a BLAKE3 output, so the +//! last byte is uniform by construction at every network size. +//! +//! 256 shards keeps a 24 GiB node at ~23 files per directory and a 1 TiB node at ~977, +//! for 1 MiB of directory inodes. The scheme and depth are recorded in `layout.json` at +//! creation so a future layout can be detected rather than silently misread. +//! +//! # Why lowercase hex names +//! +//! NTFS and default APFS fold case. Under an encoding with both cases (base64url, +//! base58) two distinct 32-byte keys can share one case-folded filename, which is a +//! silent overwrite. Hex has one case-folded form per key, and no hex string can ever +//! spell a reserved Windows device name (`CON`, `NUL`, `AUX`, `COM1`, ...) because none +//! of those letters is in `0-9a-f`. The full 64-character key stays in the filename, so +//! a `find` over the tree recovers the whole store even if the directory layer is lost. +//! +//! [`CLOSE_GROUP_SIZE`]: crate::ant_protocol::CLOSE_GROUP_SIZE -use crate::ant_protocol::XorName; +use crate::ant_protocol::{XorName, MAX_CHUNK_SIZE, XORNAME_LEN}; use crate::error::{Error, Result}; -use crate::logging::{debug, error, info, warn}; -use crate::storage::file_store::{FileStore, FileStoreConfig}; -use crate::storage::lmdb::{LmdbStorage, LmdbStorageConfig}; -use crate::storage::migration::{ - CopyReport, MigrationConfig, MigrationPhase, MigrationState, REQUIRED_REBUILDS_BEFORE_RETIRE, -}; +use crate::logging::{debug, info, trace, warn}; use crate::storage::StorageStats; -use std::collections::{BTreeMap, BTreeSet}; -use std::io::Write; +use fs2::FileExt; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeSet, HashMap, HashSet}; +use std::fs::{File, OpenOptions}; +use std::io::{ErrorKind, Read, Write}; use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; -use std::time::Duration; -use tokio_util::sync::CancellationToken; +use std::time::{Duration, Instant}; +use tokio::task::spawn_blocking; +use tokio_util::task::TaskTracker; -/// Directory name of the legacy LMDB environment, under the node root. -pub const LEGACY_ENV_DIR: &str = "chunks.mdb"; +/// Directory under the node root that holds the chunk files. +pub const CHUNKS_DIR_NAME: &str = "chunks"; -/// Suffix for a legacy environment that has been retired but not yet deleted. -pub const RETIRED_SUFFIX: &str = ".retired"; +/// Name of the layout marker written once at store creation. +pub const LAYOUT_FILE_NAME: &str = "layout.json"; -/// Written inside a chunk environment directory once it has been retired. -/// -/// The rename that moves the environment aside cannot be shown to be durable off Unix: -/// there is no way to flush a directory through the standard library, and `MoveFileEx` is -/// not documented as durable at return without a flag std does not use. So a power loss -/// can bring the directory back under its old name with its contents already deleted, and -/// a node that tried to open that would fail to start. -/// -/// This file is what makes that unambiguous, and it is *inside* the directory rather than -/// beside it so that it travels with it: a directory that reverts to its old name reverts -/// carrying its own evidence. It is created with the same create-and-flush that publishes -/// a chunk, which is documented as durable everywhere, and only after the rename has -/// already succeeded. So a directory holding it has been retired, whatever it is called, -/// and one that does not is a live environment and is opened normally. -/// -/// Deliberately not a file beside the environment. A marker that can outlive the thing it -/// describes has to be cancelled, cancellation can fail or be lost, and a stale one would -/// authorise deleting an environment that had since taken a chunk. -const RETIRED_MARKER: &str = "RETIRED"; +/// Name of the advisory single-process lock file. +const LOCK_FILE_NAME: &str = ".lock"; -/// How many times the background reaper retries deleting a retired directory. +/// Prefix that marks an in-flight write. Never a valid chunk name (chunk names are +/// exactly [`CHUNK_NAME_LEN`] lowercase hex characters, and `.` is not hex). +const TEMP_PREFIX: &str = ".tmp."; + +/// Number of shard directories. One level, `00` through `ff`. +const SHARD_COUNT: usize = 256; + +/// Length of a chunk filename: the full address in lowercase hex. +const CHUNK_NAME_LEN: usize = XORNAME_LEN * 2; + +/// How often to re-query available disk space, in seconds. /// -/// Generous, because giving up strands the disk until the next restart and the thread -/// costs nothing while it sleeps. With the backoff below this keeps trying for about a -/// day. -const RETIRED_DELETE_ATTEMPTS: u32 = 60; +/// Matches the LMDB store's cadence so the capacity predicate behaves identically +/// for callers that only ask "is there room at all". +const DISK_CHECK_INTERVAL_SECS: u64 = 5; -/// Base wait between those attempts, multiplied by the attempt number up to the cap. -const RETIRED_DELETE_BACKOFF: Duration = Duration::from_secs(10); +/// Allocation granularity assumed when charging a pending write against free space. +/// +/// Every filesystem we support allocates in units of at least 4 KiB, so a write of +/// `n` bytes consumes at least `ceil(n / 4096) * 4096`. One extra unit covers the +/// directory entry and inode. +const ALLOC_UNIT: u64 = 4096; -/// The longest the reaper waits between attempts. -const RETIRED_DELETE_BACKOFF_MAX: Duration = Duration::from_secs(30 * 60); +/// How many times a publish retries a transient Windows sharing violation. +const RENAME_RETRY_ATTEMPTS: u32 = 5; -/// How many retired directories may be waiting to be deleted before the node stops -/// finding new names for them. Far more than a node should ever accumulate. -const MAX_TOMBSTONES: u32 = 64; +/// Base backoff between those retries; the wait grows linearly with the attempt. +const RENAME_RETRY_BACKOFF: Duration = Duration::from_millis(20); -/// The legacy environment's data file. Its presence is what says a node still has one. -const LEGACY_DATA_FILE: &str = "data.mdb"; +/// Longest absolute path a chunk file may need, checked once at open. +/// +/// Windows caps a non-verbatim path at `MAX_PATH` (260) including the terminating NUL. +/// Rust's standard library transparently switches to the `\\?\` verbatim form for long +/// absolute paths, so this is a warning rather than a hard failure, but an operator who +/// buries the node root ten directories deep should hear about it before the first write +/// fails rather than after. +#[cfg(windows)] +const WINDOWS_PATH_WARN_LEN: usize = 240; + +/// The on-disk layout marker. +/// +/// Written once when the store directory is created and read on every subsequent open. +/// Nothing in this survey of comparable stores (IPFS flatfs, Storj, borgbackup) shipped +/// an in-place re-sharder, and all three paid for it. Recording the scheme costs one +/// small file and is the difference between changing the default later and never being +/// able to. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct StoreLayout { + /// Marker schema version. A store written by a newer schema is refused. + pub schema: u32, + /// How a chunk address maps to a shard directory. + pub scheme: String, + /// How many hex characters of the address name the shard directory. + pub shard_chars: u8, + /// How many directory levels of sharding. + pub depth: u8, + /// How a chunk address maps to a filename. + pub name_encoding: String, +} -/// How many times retirement retries taking sole ownership of the legacy handle before -/// giving up for this tick. -const RETIRE_UNWRAP_ATTEMPTS: u32 = 20; +/// Marker schema this build writes and understands. +const LAYOUT_SCHEMA: u32 = 1; +/// Shard scheme this build implements: the trailing hex characters of the address. +const LAYOUT_SCHEME_SUFFIX_HEX: &str = "suffix-hex"; +/// Filename encoding this build implements. +const LAYOUT_NAME_LOWER_HEX: &str = "lower-hex"; -/// How long to wait between those attempts. -const RETIRE_UNWRAP_BACKOFF: Duration = Duration::from_millis(100); +impl Default for StoreLayout { + fn default() -> Self { + Self { + schema: LAYOUT_SCHEMA, + scheme: LAYOUT_SCHEME_SUFFIX_HEX.to_string(), + shard_chars: 2, + depth: 1, + name_encoding: LAYOUT_NAME_LOWER_HEX.to_string(), + } + } +} -/// How many chunks the verification pass checks between progress lines. -const VERIFY_LOG_EVERY: u64 = 2000; +impl StoreLayout { + /// Return an error unless this build can read a store written with this layout. + fn check_supported(&self) -> Result<()> { + if self.schema > LAYOUT_SCHEMA { + return Err(Error::Storage(format!( + "Chunk store layout schema {} is newer than this build understands ({LAYOUT_SCHEMA}). \ + Refusing to open rather than misread the store.", + self.schema + ))); + } + if self.scheme != LAYOUT_SCHEME_SUFFIX_HEX { + return Err(Error::Storage(format!( + "Chunk store uses shard scheme '{}', this build implements '{LAYOUT_SCHEME_SUFFIX_HEX}'", + self.scheme + ))); + } + if self.shard_chars != 2 || self.depth != 1 { + return Err(Error::Storage(format!( + "Chunk store uses {} shard characters at depth {}, this build implements 2 at depth 1", + self.shard_chars, self.depth + ))); + } + if self.name_encoding != LAYOUT_NAME_LOWER_HEX { + return Err(Error::Storage(format!( + "Chunk store names files with '{}', this build implements '{LAYOUT_NAME_LOWER_HEX}'", + self.name_encoding + ))); + } + Ok(()) + } +} -/// How many per-key critical sections the facade keeps. +/// What the store can say about free space right now. +/// +/// Three answers, because deciding how long to stand down needs the distinction: a full +/// disk is a standing condition worth waiting minutes on, while a failed query may have +/// cleared by the next attempt and must not be treated as one. /// -/// Keyed on the address's LAST byte, for the same reason the shard directories are: a -/// node's keys share their leading bytes, so lanes keyed on the first byte would all -/// collapse into one. -const KEY_LOCK_LANES: usize = 256; +/// Lived alongside the LMDB store until that was removed. It was never about LMDB. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CapacityVerdict { + /// Available space is at or above the configured reserve. That is what the query + /// establishes, and possibly from the TTL cache, not a promise the next write succeeds. + Writable, + /// Available space is below the configured reserve. + Full, + /// The query itself failed, so nothing is known about available space. + Unknown, +} /// Configuration for [`ChunkStore`]. #[derive(Debug, Clone)] pub struct ChunkStoreConfig { - /// Node root directory. + /// Node root directory. The store lives at `{root_dir}/chunks/`. pub root_dir: PathBuf, /// Verify `BLAKE3(content) == address` on read. pub verify_on_read: bool, - /// Explicit LMDB map size cap in bytes, used only while a legacy environment exists. - /// - /// Dies with LMDB. Kept so an operator's existing `storage.db_size_gb` still means - /// what it meant during the bridge. - pub max_map_size: usize, - /// Minimum free disk space to preserve on the storage partition. + /// Free bytes to keep on the storage partition. Writes are refused below this. pub disk_reserve: u64, - /// Migration controls. - pub migration: MigrationConfig, } impl Default for ChunkStoreConfig { @@ -113,16 +205,13 @@ impl Default for ChunkStoreConfig { Self { root_dir: PathBuf::from(".ant/chunks"), verify_on_read: true, - max_map_size: 0, disk_reserve: crate::storage::DEFAULT_DISK_RESERVE, - migration: MigrationConfig::default(), } } } impl ChunkStoreConfig { - /// A test-friendly default with the disk reserve disabled, so unit tests do not - /// depend on the host having spare gigabytes. + /// The shipped defaults with the disk reserve removed, for tests on small volumes. #[cfg(any(test, feature = "test-utils"))] #[must_use] pub fn test_default() -> Self { @@ -133,674 +222,878 @@ impl ChunkStoreConfig { } } -/// The legacy environment and the keys only it still holds. -#[derive(Clone)] -struct Legacy { - /// The LMDB handle. - lmdb: Arc, - /// Keys in the legacy environment that are **not** in the file store. - /// - /// Kept in memory so the union view costs nothing on the hot paths: `exists` and - /// `current_chunks` never touch LMDB, and `all_keys` merges two already-sorted - /// sequences. It is derived at open (LMDB keys minus file keys) and maintained by - /// every write, copy and delete. - only: Arc>>, - /// Writes that have started and whose outcome is not yet known. - /// - /// A write into the legacy environment runs on a blocking thread that outlives the - /// future waiting for it, so a shutdown can leave the environment holding a chunk - /// while nothing ran to record it. A key in neither view is what retirement destroys, - /// so every write announces itself here first. - /// - /// Deliberately NOT part of what the node says it holds. This is a note to itself - /// that something is in flight, not a claim: `exists`, `all_keys`, the commitment, the - /// quote count and the pruner all ignore it. It vetoes retirement, and the driver - /// resolves each entry against what is actually on disk. +/// Outcome of a single write attempt, used to keep the duplicate accounting honest. +enum PutOutcome { + /// The chunk was newly published. + New, + /// The chunk was already on disk. + Duplicate, +} + +/// Snapshot of free space, plus what has been written since it was taken. +#[derive(Debug)] +struct CapacitySnapshot { + /// When `available` was measured. `None` means never. + measured_at: Option, + /// Free bytes reported by the filesystem at `measured_at`. + available: u64, + /// Bytes published since `measured_at`, charged against `available`. + /// + /// Cleared by a fresh measurement, which already accounts for them. + written_since: u64, + /// Bytes reserved by writes that have not landed yet. + /// + /// Deliberately **not** cleared by a measurement: a `statvfs` taken while writes are + /// in flight reports space those writes are about to consume, so forgetting their + /// reservations at that moment would hand the same bytes out twice. That is precisely + /// the over-admission the reservation exists to prevent. + in_flight: u64, +} + +/// Size-aware free-space predicate with a short-lived cache. +/// +/// Free bytes alone stopped being a sufficient answer the moment chunks became files: +/// a caller wants to know whether *this* write fits, not whether the disk is non-empty. +/// The cache keeps the common case at one `statvfs` per interval while staying correct +/// under a burst, because bytes written since the measurement are charged against it. +#[derive(Debug)] +struct CapacityGuard { + /// Directory whose partition is measured. + dir: PathBuf, + /// Free bytes to keep unused. + reserve: u64, + /// The cached measurement. + snapshot: parking_lot::Mutex, +} + +impl CapacitySnapshot { + /// Free bytes, less everything written or promised since the measurement. + fn free_estimate(&self) -> u64 { + self.available + .saturating_sub(self.written_since) + .saturating_sub(self.in_flight) + } +} + +impl CapacityGuard { + /// Create a guard over the partition hosting `dir`. + fn new(dir: PathBuf, reserve: u64) -> Self { + Self { + dir, + reserve, + snapshot: parking_lot::Mutex::new(CapacitySnapshot { + measured_at: None, + available: 0, + written_since: 0, + in_flight: 0, + }), + } + } + + /// Bytes actually consumed on disk by a payload of `len` bytes. + fn charge(len: u64) -> u64 { + // Round the payload up to the allocation unit, then add one unit for the + // directory entry and inode. + len.div_ceil(ALLOC_UNIT) + .saturating_mul(ALLOC_UNIT) + .saturating_add(ALLOC_UNIT) + } + + /// Free bytes right now, or `None` if the question could not be answered. + /// + /// Deliberately separate from [`Self::measure`], which folds a failure into an error + /// the caller cannot tell from "below the reserve". + fn measure_available(&self) -> Option { + let mut snapshot = self.snapshot.lock(); + match self.measure(&mut snapshot) { + Ok(()) => Some(snapshot.free_estimate()), + Err(_) => None, + } + } + + /// Query the filesystem and refresh the snapshot. + fn measure(&self, snapshot: &mut CapacitySnapshot) -> Result<()> { + let available = fs2::available_space(&self.dir) + .map_err(|e| Error::Storage(format!("Failed to query available disk space: {e}")))?; + snapshot.available = available; + // Reservations survive: their bytes are not on the platter yet, so the fresh + // measurement does not include them. + snapshot.written_since = 0; + snapshot.measured_at = Some(Instant::now()); + Ok(()) + } + + /// Test `needed` against the snapshot, refreshing it if it is stale or short. /// - /// How many writes were made without a rollback copy of the chunk in the environment. + /// Only *passing* results are cached, so a low-space condition is rechecked on every + /// call and freed space is noticed promptly. + fn admit(&self, snapshot: &mut CapacitySnapshot, needed: u64) -> Result<()> { + let want = self.reserve.saturating_add(Self::charge(needed)); + + let cache_fresh = snapshot + .measured_at + .is_some_and(|t| t.elapsed().as_secs() < DISK_CHECK_INTERVAL_SECS); + if cache_fresh && snapshot.free_estimate() >= want { + return Ok(()); + } + + self.measure(snapshot)?; + if snapshot.free_estimate() < want { + // Do not cache a failing result: `measured_at` is left set so the next call + // still re-measures, because the branch above only short-circuits a pass. + return Err(Error::Storage(format!( + "Insufficient disk space: {:.2} GiB available, {:.2} GiB reserve required. \ + Free disk space or increase the partition to continue storing chunks.", + bytes_to_gib(snapshot.free_estimate()), + bytes_to_gib(self.reserve), + ))); + } + Ok(()) + } + + /// Drop the cached measurement so the next question hits the filesystem. + fn invalidate(&self) { + let mut snapshot = self.snapshot.lock(); + snapshot.measured_at = None; + snapshot.written_since = 0; + } + + /// Return `Ok(())` if a write of `needed` bytes would fit. Charges nothing. + fn check(&self, needed: u64) -> Result<()> { + let mut snapshot = self.snapshot.lock(); + self.admit(&mut snapshot, needed) + } + + /// Admit a write of `needed` bytes and charge it in the same critical section. /// - /// The rollback copy is best effort by design, and the ADR says so: a bridging node - /// whose environment has no reusable page keeps serving from files and simply has no - /// second copy to roll back to. What was missing was any way to ask how often that - /// happened. One `warn!` per chunk is not an answer to "how many nodes on this fleet - /// are actually keeping a rollback copy", which is the question the second release - /// turns on, and on a node with no free pages it is also a line per chunk forever. + /// Checking and charging separately is the bug this exists to prevent: dozens of + /// protocol handlers can each pass against the same cached measurement before any of + /// them has written a byte, and collectively cross the reserve. /// - /// Writes, not distinct chunks: two attempts at one address count twice, and a later - /// attempt that succeeds does not count back down. Counted before the file half runs, - /// so a write that then fails outright is counted too. Keeping a set of addresses - /// instead would be exact and would also mean holding millions of them in memory to - /// answer a question that a rate answers. Read it as "this node is failing to keep - /// rollback copies, this often", not as a chunk count. - skipped_rollback_copies: Arc, - - /// Counted, not a set, for the reason the file store's `writing` map is counted. - /// Cancellation can release the facade's key lane while the blocking half survives, so - /// a second write for the same key can start behind the first. With one entry between - /// them, whichever returned first would clear it while the other was still queued, and - /// a delete arriving in that window would see no announcement, skip draining the - /// environment, and let the surviving write land afterwards and put the key back. - pending: Arc>>, + /// The returned [`Reservation`] settles itself when dropped, so a caller whose future + /// is dropped mid-write cannot strand it. Nothing else ever decrements the in-flight + /// count, so a stranded reservation would be permanent, and enough of them would make + /// an empty disk look full until the process restarted. + fn reserve(self: &Arc, needed: u64) -> Result { + { + let mut snapshot = self.snapshot.lock(); + self.admit(&mut snapshot, needed)?; + snapshot.in_flight = snapshot.in_flight.saturating_add(Self::charge(needed)); + } + Ok(Reservation { + capacity: Arc::clone(self), + bytes: needed, + settled: false, + }) + } + + /// Give back a reservation whose write did not happen. + fn release(&self, needed: u64) { + let mut snapshot = self.snapshot.lock(); + snapshot.in_flight = snapshot.in_flight.saturating_sub(Self::charge(needed)); + } + + /// Turn a reservation into bytes that are now on disk. + fn commit_reservation(&self, needed: u64) { + let charge = Self::charge(needed); + let mut snapshot = self.snapshot.lock(); + snapshot.in_flight = snapshot.in_flight.saturating_sub(charge); + snapshot.written_since = snapshot.written_since.saturating_add(charge); + } + + /// Credit a completed delete back to the cached measurement. + fn record_removed(&self, len: u64) { + let mut snapshot = self.snapshot.lock(); + snapshot.written_since = snapshot.written_since.saturating_sub(Self::charge(len)); + } } -impl Legacy { - /// Note that a chunk went to files alone, and say whether to log it. - /// - /// Throttled by powers of ten. The condition is usually all-or-nothing, so the first - /// few lines say it started and the later ones say it is still going without becoming - /// the log. - fn note_skipped_rollback_copy(&self) -> Option { - let count = self - .skipped_rollback_copies - .fetch_add(1, std::sync::atomic::Ordering::Relaxed) - .saturating_add(1); - let round = matches!( - count, - 10 | 100 | 1_000 | 10_000 | 100_000 | 1_000_000 | 10_000_000 - ); - (count <= 3 || round).then_some(count) +/// A charged, unsettled write. +/// +/// Held by whatever is actually doing the write, so the charge is released even if the +/// caller's future is dropped and only the blocking closure survives. +struct Reservation { + /// The guard this was taken from. + capacity: Arc, + /// Payload size, before rounding. + bytes: u64, + /// Whether it has already been accounted for. + settled: bool, +} + +impl Reservation { + /// The write landed: move the charge from in-flight to written. + fn commit(mut self) { + self.capacity.commit_reservation(self.bytes); + self.settled = true; } +} - /// Announce a write into the environment, or note a second one for the same key. - fn announce(&self, address: &XorName) { - *self.pending.write().entry(*address).or_insert(0) += 1; +impl Drop for Reservation { + fn drop(&mut self) { + if !self.settled { + self.capacity.release(self.bytes); + } } +} - /// Retire one announcement, leaving any other for the same key still standing. - fn announced_write_finished(&self, address: &XorName) { - let mut pending = self.pending.write(); - let Some(count) = pending.get_mut(address) else { - return; +/// Environment variable naming a failpoint: stop after the temp file, before the rename. +#[cfg(any(test, feature = "test-utils"))] +pub const HALT_BEFORE_PUBLISH: &str = "ANT_HALT_BEFORE_PUBLISH"; + +/// Park forever at a named failpoint, once a marker says the process has reached it. +/// +/// For crash tests, which need a process to die *inside* an operation rather than at +/// whatever point a sleep in another process happened to land. The variable holds a path: +/// this writes it, so the parent knows the child is exactly here, and then waits to be +/// killed. +/// +/// Costs one environment read per write when the feature is compiled in, and the feature +/// is not in a release build. +#[cfg(any(test, feature = "test-utils"))] +pub(crate) fn halt_here_if_asked(variable: &str, reached: &Path) { + let Ok(marker) = std::env::var(variable) else { + return; + }; + // Let the first few through. A test that stops the very first write leaves a store + // with nothing successfully in it, and an assertion over what it holds then passes by + // iterating nothing. Letting some land first means the crash happens to a store that + // has real chunks in it, which is the situation worth checking. + let skip: u64 = std::env::var(HALT_AFTER) + .ok() + .and_then(|raw| raw.parse().ok()) + .unwrap_or(0); + if HALTS_SEEN.fetch_add(1, std::sync::atomic::Ordering::AcqRel) < skip { + return; + } + if let Err(e) = std::fs::write(&marker, reached.as_os_str().as_encoded_bytes()) { + // The parent waits for this file. Saying so on the way past is the difference + // between a test that fails and one that hangs until the job times out. + eprintln!("failpoint could not write its marker {marker}: {e}"); + return; + } + loop { + std::thread::sleep(Duration::from_secs(3600)); + } +} + +/// How many writes to let through before the failpoint fires. +#[cfg(any(test, feature = "test-utils"))] +pub const HALT_AFTER: &str = "ANT_HALT_AFTER"; + +/// How many times the failpoint has been reached in this process. +#[cfg(any(test, feature = "test-utils"))] +static HALTS_SEEN: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + +/// Clears a write's registration when the work finishes, however it finishes. +/// +/// Held by the blocking closure rather than by the caller, so a dropped future cannot +/// leave an entry behind, and a panic in the work cannot either. +struct WriteInFlight { + writing: Arc>>, + finished: Arc, + address: XorName, +} + +impl Drop for WriteInFlight { + fn drop(&mut self) { + let was_last = { + let mut writing = self.writing.lock(); + match writing.get_mut(&self.address) { + Some(count) if *count > 1 => { + *count -= 1; + false + } + _ => { + writing.remove(&self.address); + true + } + } }; - *count = count.saturating_sub(1); - if *count == 0 { - pending.remove(address); + // Only when this was the last one. Waking a waiter while another write for the + // same key is still queued is exactly what the count exists to prevent. + if was_last { + self.finished.notify_waiters(); } } } -/// Content-addressed chunk storage. +/// What is behind a chunk's name on disk. +/// +/// Four answers, not two, because "could not read it" must never be treated as "wrong": +/// replacing a chunk is destructive, and off Unix it truncates the file in place, so a +/// transient fault would turn a healthy sole copy into an empty one. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum StoredBytes { + /// The bytes are there and hash to the name. + Good, + /// The bytes are there and do not. + Wrong, + /// There is nothing behind the name. + Absent, + /// The question could not be answered this time. + Unreadable, +} + +/// Convert a byte count to GiB for human-readable log messages. +#[allow(clippy::cast_precision_loss)] // display only — sub-byte precision is irrelevant +fn bytes_to_gib(bytes: u64) -> f64 { + bytes as f64 / (1024.0 * 1024.0 * 1024.0) +} + +/// Content-addressed store holding one immutable file per chunk. +/// +/// The filesystem is the sole authority. The in-memory index is a cache of what the +/// directory tree already contains, rebuilt from directory entries at every open, and +/// every mutation of it mirrors a filesystem operation that has *already* completed. +/// Bitcask's issue #114 is the cautionary tale for the opposite order: an index that is +/// rebuilt at startup and then mutated in anticipation drifts, and the drift is silent. +#[derive(Debug)] pub struct ChunkStore { - /// The file store. Always present, always the write target. - files: Arc, - /// The legacy environment, until it is retired. - legacy: parking_lot::RwLock>, - /// Excludes retirement while any operation that touches the legacy environment runs. - /// - /// Reads, writes and deletes take it shared for their whole duration; retirement takes - /// it exclusively before it takes the environment away. Sole ownership of the handle - /// is not enough on its own: a read that has decided the file store cannot answer, and - /// has not yet taken a legacy handle, holds nothing and would be invisible to that - /// check. Nor would holding a handle throughout do instead, because on a busy node - /// there would always be one and retirement would never see the environment - /// unreferenced. Retirement also waits for the environment to go idle, which never - /// happens if new work can keep starting in it. - /// - /// A shared/exclusive lock states the actual requirement, and because it is fair, a - /// waiting retirement stops new work starting rather than starving behind it. - retirement: tokio::sync::RwLock<()>, - /// Where the legacy environment lives. - legacy_env_dir: PathBuf, /// Store configuration. config: ChunkStoreConfig, - /// The persisted migration marker. - state: parking_lot::RwLock, - /// One lock per shard, held across a whole logical key transition. - /// - /// The file store has its own lane locks, but those only make a single file write - /// atomic. The races that matter here span two stores and an await point: the copier - /// reads a chunk out of LMDB, the pruner deletes that chunk from both stores, and - /// then the copier's write lands and resurrects it. One critical section per key, - /// held across put, delete and copy, is what closes that. - key_locks: Vec>, + /// `{root_dir}/chunks`. + chunks_dir: PathBuf, + /// Every address whose file is published, in ascending order. + /// + /// `BTreeSet` rather than a hash set because `all_keys()` must be sorted (the + /// commitment builder truncates with `take(cap)` *before* the Merkle tree sorts, so + /// an unstable order would make the node's published commitment depend on iteration + /// luck), and because it never spikes memory while growing. + index: Arc>>, + /// One mutex per shard, serialising writers of the same address. + /// + /// LMDB gave exactly-once `put` semantics for free: the duplicate test happened + /// inside the write transaction. Two threads publishing the same address here would + /// otherwise both see an absent file, both rename, and both report "newly stored", + /// double-counting the chunk. The lane is indexed by the address's LAST byte for the + /// same reason the shard is: a node's keys share their leading bytes, so lanes keyed + /// on the first byte would all collapse into one. + write_lanes: Arc>>, + + /// One lock per shard, held across a whole logical transition for a key. + /// + /// Not the same thing as the write lanes above, which are taken inside a blocking + /// closure and make one file write atomic. These are held across await points, which is + /// what the races that matter need: a delete has to exclude a read that is deciding + /// whether to accept an offered copy, and both span an await. Without it a prune can + /// remove the file between that read and its answer, and the caller is told the chunk + /// is already held while the copy that would have replaced it is discarded. + /// + /// Indexed by the address's LAST byte, for the reason the shard is: a node's keys share + /// their leading bytes, so lanes keyed on the first would collapse into one. + key_locks: Arc>>, + /// Operation counters, same shape as the LMDB store reported. + stats: parking_lot::RwLock, + /// Which of the 256 shard directories are known to exist, so a steady-state write + /// does not pay a `create_dir_all` syscall. + shards_present: Arc>, + /// Indexed chunks this store currently cannot read. + /// + /// Held back from everything the node says it has, while the files themselves are + /// left alone. See [`Self::mark_suspect`]. + suspect: Arc>>, + /// Indexed chunks a read has proven do not match their name. + /// + /// Separate from the above because they clear differently. Not being able to read a + /// file is a question a later read answers; bytes that are wrong stay wrong however + /// often they are read, and only a repair or a removal settles it. A raw read that + /// does not hash anything must not take a chunk out of this set. + known_wrong: Arc>>, + /// Addresses this store is part-way through writing. + /// + /// Every mutation registers here before it spawns its blocking work and clears the + /// entry *inside* that work, so a caller whose future is dropped cannot skip the + /// clearing while the write itself goes on to land. That is the difference that + /// matters: the blocking half is not cancelled with the future, so anything the + /// future was going to do afterwards is not a record of what happened. + /// + /// It lets a delete queue behind the exact write it would otherwise race, rather than + /// behind every write this store has in flight. + /// + /// Counted, not a set. Cancellation can release the facade's key lane while the + /// blocking half survives, so a second write for the same key can start behind the + /// first. With one entry between them, whichever finished first would remove it and a + /// waiter would be told the key is free while the other was still queued. + writing: Arc>>, + /// Woken when [`Self::writing`] loses its last entry for a key. + write_finished: Arc, + /// Size-aware free-space predicate. + capacity: Arc, + /// Monotonic counter that makes temp filenames unique within this store. + temp_seq: AtomicU64, + /// Random per-instance discriminator for temp filenames. + nonce: u32, + /// Held for the store's lifetime. Startup fails without it. + /// + /// Shared rather than owned so the blocking work that depends on it can hold a lease + /// of its own: that work outlives the future that spawned it, and a cancelled caller + /// releasing the lock would leave it writing into a directory another process had + /// just been let into. + lock: Arc, + /// Tracks every blocking task, so [`ChunkStore::wait_idle`] can wait for writes that + /// outlived their awaiting future. + blocking_tracker: TaskTracker, + /// Test-only gate read-acquired at the top of the put blocking closure. + /// + /// Tests hold the write half to park an in-flight write on the blocking pool, which + /// is the shape a `select!` losing to a shutdown token leaves behind. + #[cfg(test)] + test_put_gate: Arc>, + + /// Test-only: parks a put after it has taken the key's lane and before it registers + /// itself as in flight. + /// + /// Asynchronous, unlike the gate above. That one is taken inside a blocking closure on + /// its own thread; this one is taken on the runtime, so a synchronous lock here would + /// block the executor and the test would deadlock instead of observing anything. + /// + /// A separate gate from the one above, because that one sits inside the blocking + /// closure, which is after registration. The window this opens is the one the key lane + /// exists for: a put that a delete's wait cannot see yet, because there is nothing to + /// see. Without a hook here, a test cannot tell a delete blocked by the lane from a + /// delete blocked by the wait, and so cannot show the lane is doing anything. + #[cfg(test)] + test_pre_registration_gate: Arc>, + + /// Test-only: how many puts have reached that gate. + /// + /// So a test can wait for the put to be parked rather than sleeping and hoping. A sleep + /// makes the staging a guess, and a guess in a test that is meant to be deterministic + /// is a flake waiting for a loaded machine. + #[cfg(test)] + test_reached_pre_registration: Arc, } impl ChunkStore { - /// Open the store under `config.root_dir`. + /// Open (or create) the store at `{root_dir}/chunks/`. /// - /// Opens the legacy environment only if one is already on disk. A fresh node never - /// creates one, so it never pays for a memory map it will not use. + /// Sweeps orphaned temp files, then rebuilds the index from directory entries. + /// The scan reads names only: it never `stat`s an entry and never reads a chunk. /// /// # Errors /// - /// Returns [`Error::Storage`] if either store cannot be opened. + /// Returns [`Error::Storage`] if the directory cannot be created, the layout marker + /// is unreadable or describes a layout this build does not implement, or the scan + /// fails. pub async fn new(config: ChunkStoreConfig) -> Result { - let files = Arc::new( - FileStore::new(FileStoreConfig { - root_dir: config.root_dir.clone(), - verify_on_read: config.verify_on_read, - disk_reserve: config.disk_reserve, - }) - .await?, - ); - - // Before anything looks at the legacy environment: a directory carrying its own - // retirement mark is the remains of a removal a power loss interrupted, and is - // moved aside rather than opened. - let openable = finish_interrupted_retirement(&config.root_dir); - let legacy_env_dir = config.root_dir.join(LEGACY_ENV_DIR); - let legacy = - if openable == LiveEnvironment::WhateverIsOnDisk && legacy_present(&config.root_dir)? { - Some(Self::open_legacy(&config, &files).await?) - } else { - None - }; - - let phase = if legacy.is_some() { - MigrationPhase::Bridging - } else { - MigrationPhase::FilesOnly - }; - let mut state = MigrationState::load_or_create(&config.root_dir, phase); - - // The filesystem is the authority on whether a legacy environment exists; the - // marker only records decisions. Reconcile rather than trust. - if legacy.is_none() && state.phase != MigrationPhase::FilesOnly { - info!("No legacy chunk environment on disk; the migration is already complete"); - state.phase = MigrationPhase::FilesOnly; - if let Err(e) = state.save(&config.root_dir) { - warn!("Could not persist the migration marker: {e}"); - } - } else if legacy.is_some() - && state.phase == MigrationPhase::Committed - && files.current_chunks().unwrap_or(0) < state.kept_key_count - { - // The marker says this node already settled on what it would keep, but the - // file store holds less than it recorded keeping. Something outside the node - // changed the data directory, and trusting the marker here would skip the - // copier, the shed rules and their rank checks on the way to deleting the - // legacy environment. The filesystem wins. - warn!( - "The migration marker says this node kept {} chunk(s) but the file store \ - holds {}. Restarting the migration from the copying stage.", - state.kept_key_count, - files.current_chunks().unwrap_or(0) - ); - state.phase = MigrationPhase::Bridging; - state.committed_at_unix = None; - state.rebuilds_since_commit = 0; - if let Err(e) = state.save(&config.root_dir) { - warn!("Could not persist the migration marker: {e}"); - } - } else if legacy.is_some() && state.phase == MigrationPhase::FilesOnly { - warn!( - "The migration marker says this node is done but {} is still on disk. \ - Resuming the bridge.", - legacy_env_dir.display() - ); - state.phase = MigrationPhase::Bridging; - if let Err(e) = state.save(&config.root_dir) { - warn!("Could not persist the migration marker: {e}"); - } - } - - let store = Self { - files, - legacy: parking_lot::RwLock::new(legacy), - retirement: tokio::sync::RwLock::new(()), - legacy_env_dir, - config, - state: parking_lot::RwLock::new(state), - key_locks: std::iter::repeat_with(|| tokio::sync::Mutex::new(())) - .take(KEY_LOCK_LANES) - .collect(), - }; - - let (file_keys, legacy_keys) = store.split_counts(); - info!( - "Chunk store ready: {file_keys} chunks in files, {legacy_keys} still only in the \ - legacy environment, phase {:?}", - store.migration_phase() - ); - Ok(store) - } - - /// Open the legacy environment and work out which keys only it holds. - async fn open_legacy(config: &ChunkStoreConfig, files: &FileStore) -> Result { - let (lmdb, legacy_keys) = Self::open_legacy_env(config).await?; - Ok(Legacy { - lmdb, - only: Arc::new(parking_lot::RwLock::new(Self::keys_only_in_legacy( - &legacy_keys, - files, - ))), - skipped_rollback_copies: Arc::new(std::sync::atomic::AtomicU64::new(0)), - pending: Arc::new(parking_lot::RwLock::new(BTreeMap::new())), + // Deliberately nothing about the old chunk store here. Clearing up after the storage + // migration happens once, in `NodeBuilder::build`, and only after this constructor + // has succeeded: what it deletes are directories whose chunks are in this store, so + // it may not run until this store is open. Asking here as well would put that + // decision in front of itself. + + let chunks_dir = config.root_dir.join(CHUNKS_DIR_NAME); + std::fs::create_dir_all(&chunks_dir).map_err(|e| { + Error::Storage(format!( + "Failed to create chunk store directory {}: {e}", + chunks_dir.display() + )) + })?; + + check_path_budget(&chunks_dir); + + let layout = read_or_write_layout(&chunks_dir)?; + layout.check_supported()?; + + // Startup fails without it, so from here this process is the only one using this + // directory and an interrupted write can only be its own. + let lock = acquire_store_lock(&chunks_dir)?; + + let scan_dir = chunks_dir.clone(); + // The scan holds the lease itself. It sweeps interrupted writes on the strength of + // being alone here, and it runs on a thread that outlives this future: a + // cancelled startup that released the lock would leave it sweeping a directory + // another process had just been let into. + let scan_lease = Arc::clone(&lock); + // The node root as well as the chunk tree. The scan sweeps interrupted writes + // under `chunks/`, which covers the layout marker's temporary because that lives + // there; the migration marker's lives in the root, where nothing looked. + let root = config.root_dir.clone(); + let scan = spawn_blocking(move || { + let _lease = scan_lease; + sweep_marker_temps(&root); + scan_store(&scan_dir) }) - } + .await + .map_err(|e| Error::Storage(format!("Chunk store scan task failed: {e}")))??; - /// Open the legacy environment and read every key in it. - /// - /// Split from the diff against the file store because the two want different timing: - /// this is slow and safe to do at any moment, the diff has to be the last thing before - /// the handle is installed. - async fn open_legacy_env( - config: &ChunkStoreConfig, - ) -> Result<(Arc, Vec)> { - let lmdb = Arc::new( - LmdbStorage::new(LmdbStorageConfig { - root_dir: config.root_dir.clone(), - verify_on_read: config.verify_on_read, - max_map_size: config.max_map_size, - disk_reserve: config.disk_reserve, - }) - .await?, - ); - // From here it never grows. Two stores on one disk each measure the same free - // space and neither knows what the other is about to spend, so a chunk written to - // both can be admitted twice against one lot of headroom and the pair can cross - // the reserve together. Pinned, this one writes only from pages it already has, - // so the file store's accounting is the only claim on free disk. - // - // What it costs is that the rollback copy is made only when the environment has - // room of its own. That is the right way round: the copy exists to make a fleet - // rollback survivable, not to be the write that has to succeed, and an - // environment this migration exists to delete should not be taking new disk to - // hold a second copy of something the file store already has. - lmdb.pin_growth().await?; - let legacy_keys = lmdb.all_keys().await?; - Ok((lmdb, legacy_keys)) - } + let ScanResult { + keys, + shards_present, + swept_temps, + skipped, + } = scan; - /// Which of `legacy_keys` the file store does not have. - /// - /// In memory, no I/O: the file store answers from its index. Cheap enough to redo - /// immediately before installing a handle, which is the point. A key that lost its - /// file while the environment was being read must be in this set, or nothing will - /// look for it again and retirement will destroy the copy that is left. - fn keys_only_in_legacy(legacy_keys: &[XorName], files: &FileStore) -> BTreeSet { - legacy_keys - .iter() - .filter(|key| !files.is_indexed(key)) - .copied() - .collect() - } + let key_count = keys.len(); + // Build from a sorted vector: bulk-building packs every B-tree node to its + // capacity, where repeated `insert` converges on ~68% fill for the same keys. + let index: BTreeSet = keys.into_iter().collect(); - /// Take the critical section for one key. - async fn key_lock(&self, address: &XorName) -> Option> { - let lane = address.last().copied().unwrap_or(0) as usize; - match self.key_locks.get(lane) { - Some(lock) => Some(lock.lock().await), - None => None, + if swept_temps > 0 { + info!("Chunk store: removed {swept_temps} orphaned temporary file(s) from interrupted writes"); } - } + if skipped > 0 { + warn!("Chunk store: ignored {skipped} directory entr(ies) that are not chunk files"); + } + info!( + "Chunk store open at {} ({key_count} chunks)", + chunks_dir.display() + ); - /// A cheap clone of the legacy handle, or `None` once it is retired. - fn legacy(&self) -> Option { - self.legacy.read().clone() - } + let capacity = Arc::new(CapacityGuard::new(chunks_dir.clone(), config.disk_reserve)); - /// `(chunks in files, chunks only in the legacy environment)`. - fn split_counts(&self) -> (u64, u64) { - // Legacy first, for the reason given on `exists`: a key mid-copy is then counted - // twice for an instant rather than not at all, and over-reporting what the node - // holds is the safe direction for every caller of `current_chunks`. - let legacy = self - .legacy() - .map_or(0, |l| l.only.read().len().try_into().unwrap_or(u64::MAX)); - let files = self.files.current_chunks().unwrap_or(0); - (files, legacy) + Ok(Self { + config, + chunks_dir, + index: Arc::new(parking_lot::RwLock::new(index)), + key_locks: Arc::new( + std::iter::repeat_with(|| tokio::sync::Mutex::new(())) + .take(SHARD_COUNT) + .collect(), + ), + write_lanes: Arc::new( + std::iter::repeat_with(|| parking_lot::Mutex::new(())) + .take(SHARD_COUNT) + .collect(), + ), + stats: parking_lot::RwLock::new(StorageStats::default()), + shards_present: Arc::new(parking_lot::Mutex::new(shards_present)), + suspect: Arc::new(parking_lot::RwLock::new(HashSet::new())), + known_wrong: Arc::new(parking_lot::RwLock::new(HashSet::new())), + writing: Arc::new(parking_lot::Mutex::new(HashMap::new())), + write_finished: Arc::new(tokio::sync::Notify::new()), + capacity, + temp_seq: AtomicU64::new(0), + nonce: rand::random(), + lock, + blocking_tracker: TaskTracker::new(), + #[cfg(test)] + test_put_gate: Arc::new(parking_lot::RwLock::new(())), + #[cfg(test)] + test_pre_registration_gate: Arc::new(tokio::sync::RwLock::new(())), + #[cfg(test)] + test_reached_pre_registration: Arc::new(std::sync::atomic::AtomicU64::new(0)), + }) } /// Store a chunk. /// - /// While a legacy environment exists and dual-writing is on, the chunk goes there - /// **first**. A chunk uploaded during the bridge to holders that all revert to a - /// pre-migration build would otherwise be gone from every one of them, and that is - /// real client data, not a replica. + /// On Unix, publishing is a rename within the destination directory, so the final name + /// can never appear on partial content: the name *is* the hash, and the content is + /// fully written and flushed before the name exists. Off Unix there is no rename, for + /// the reason `publish_in_place` gives (it is compiled only on those platforms, so this + /// is not a link), and a partial file can wear a real name; that + /// is why a duplicate is read and compared rather than trusted. /// /// # Returns /// - /// `true` if the chunk was newly stored, `false` if either store already had it. + /// `true` if the chunk was newly stored, `false` if it was already present. /// /// # Errors /// - /// Returns [`Error::Storage`] if the content does not hash to `address`, the disk is - /// too full, or the write fails. + /// Returns [`Error::Storage`] if the content does not hash to `address`, the disk + /// is too full, or the write fails. pub async fn put(&self, address: &XorName, content: &[u8]) -> Result { - // Shared, for the reason given on the field: retirement waits for the legacy - // environment to go idle, and a write that keeps starting new work in it while - // that wait runs makes the wait unbounded. It also stops a write inserting a key - // into the legacy-only set after the gates have approved the set that may go. - let _using_legacy = self.retirement.read().await; + // The key's whole transition, not just the part that touches the disk. Registering + // the write is what a delete waits for, and everything before that registration + // happens outside it: validation, the duplicate read, the capacity reservation. A + // put that got that far before a delete arrived would otherwise be invisible to the + // delete's wait, register while the delete was already committed to going ahead, + // and publish afterwards. The node would then hold a chunk it had decided to prune. let _lane = self.key_lock(address).await; - let legacy = self.legacy(); - let already_in_legacy = legacy - .as_ref() - .is_some_and(|l| l.only.read().contains(address)); - - let mut dual_written = false; - if let Some(ref l) = legacy { - if self.config.migration.dual_write_legacy && !already_in_legacy { - // The legacy store's own verdict, not the file store's. It accounts for - // pages it can reuse internally, which is the right question for a write - // into it and the wrong one for the file about to be written. A full - // legacy store must not fail a put the file store can serve: the copy is - // there to make a fleet rollback survivable, and losing that for one - // chunk is much better than refusing the chunk. - if l.lmdb.capacity_verdict() == crate::storage::CapacityVerdict::Full { - // Counted here as well as on the failure path below, and this is the - // one that matters: a pinned environment with no reusable page answers - // Full for every chunk, so on the node most affected this is the whole - // of the skipping and the other path never runs at all. - if let Some(count) = l.note_skipped_rollback_copy() { - warn!( - migration_event = "no_rollback_copy", - skipped = count, - "Legacy chunk environment is full; storing {} in files only. A \ - rollback to a pre-migration build would not have this chunk, \ - and {count} write(s) on this node have now gone without a \ - rollback copy.", - hex::encode(address) - ); - } else { - debug!( - "Legacy chunk environment is full; storing {} in files only.", - hex::encode(address) - ); - } - } else { - // Announced BEFORE the write, not after it. The write runs on a - // blocking thread that outlives this future: a shutdown that drops the - // caller mid-way can leave the environment holding a chunk while - // nothing here ever ran to record it, and a key in neither view is - // what retirement destroys. In the in-flight note rather than the key - // set, because until the write returns this node does not hold the - // chunk and must not say it does. - l.announce(address); - // Best effort, and only best effort. The verdict above is optimistic - // by design: LMDB can still refuse a write for fragmentation, pages - // pinned by a long read, or a copy-on-write B-tree split. Propagating - // that would let a store this node is in the middle of abandoning - // reject paid chunks the file store has ample room for, for the whole - // bridge period. The chunk's own validity is not at stake here; the - // file store checks the content address itself. - match l.lmdb.put(address, content).await { - Ok(_) => dual_written = true, - Err(e) => { - if let Some(count) = l.note_skipped_rollback_copy() { - warn!( - migration_event = "no_rollback_copy", - skipped = count, - "Could not also write {} to the legacy environment: \ - {e}. Storing it in files only. A rollback to a \ - pre-migration build would not have this chunk, and \ - {count} write(s) on this node have now gone without a \ - rollback copy.", - hex::encode(address) - ); - } - } - } - } + let computed = crate::client::compute_address(content); + if computed != *address { + return Err(Error::Storage(format!( + "Content address mismatch: expected {}, computed {}", + hex::encode(address), + hex::encode(computed) + ))); + } + // The read path refuses anything over the ceiling, so writing one would create a + // file the store could never read back and could never repair. + if content.len() > MAX_CHUNK_SIZE { + return Err(Error::Storage(format!( + "Chunk {} is {} bytes, over the {MAX_CHUNK_SIZE} byte maximum", + hex::encode(address), + content.len() + ))); + } + + // An indexed name is not proof of the bytes under it. The index is built from + // names, by the startup scan and by a completed publish, and a name can outlive + // what it points at: off Unix a chunk is created under its final name before its + // bytes are written, so a crash leaves a short file wearing a real name, and rot + // leaves a full-length one. Answering "already have it" to the copy that would fix + // either is how a node discards its own repair and is never offered another. + // + // So the bytes decide. Checked before the reservation below, so re-storing a chunk + // this node already holds stays a no-op on a full disk. + if self.index.read().contains(address) { + if let Some(answer) = self.settle_indexed_duplicate(address, content).await { + return answer; } } - let stored_in_files = match self.files.put(address, content).await { - Ok(stored) => stored, - Err(e) => { - // The bytes reached LMDB but not the file store. Record the key as - // legacy-only so the union still finds it and the copier retries later; - // without this the node would hold a chunk it could not serve. + let len = content.len() as u64; + // Reserved after the duplicate test so re-storing an existing chunk stays a + // harmless no-op on a full disk, matching the LMDB store's ordering. + let reservation = self.capacity.reserve(len)?; + + let shard = self.chunks_dir.join(shard_name(address)); + let final_path = shard.join(hex::encode(address)); + let temp_path = shard.join(self.next_temp_name()); + let payload = content.to_vec(); + let lanes = Arc::clone(&self.write_lanes); + let index = Arc::clone(&self.index); + let shards_present = Arc::clone(&self.shards_present); + let chunks_dir = self.chunks_dir.clone(); + let lane = shard_index(address); + let key = *address; + #[cfg(test)] + let test_put_gate = Arc::clone(&self.test_put_gate); + // Registered before the work is spawned and cleared by the work itself, so a + // caller that goes away cannot leave a delete free to race this publish. + // Test-only: the window between taking the lane and being visible to a delete. + #[cfg(test)] + { + self.test_reached_pre_registration + .fetch_add(1, std::sync::atomic::Ordering::AcqRel); + drop(self.test_pre_registration_gate.read().await); + } + let in_flight = self.begin_write(address); + // And the lease, for the same reason the scan holds it: this thread writes into a + // directory whose exclusivity the lock is what establishes, and it can outlive + // the last owner of the store. + let lease = Arc::clone(&self.lock); + let known_wrong = Arc::clone(&self.known_wrong); + let suspect = Arc::clone(&self.suspect); + + let outcome = self + .blocking_tracker + .spawn_blocking(move || -> Result { + let _in_flight = in_flight; + let _lease = lease; + // Test-only: parks here while a test holds the write half. + #[cfg(test)] + let _test_put_gate = test_put_gate.read(); + let _lane = lanes.get(lane).map(parking_lot::Mutex::lock); + // `mkdir` plus a directory flush are syscalls, so they belong here and + // not on a runtime worker. + ensure_shard_dir(&chunks_dir, &shard, lane, &shards_present)?; + let outcome = match publish(&temp_path, &final_path, &payload, &shard) { + Ok(outcome) => outcome, + Err(PublishFailed { error, left_behind }) => { + // A publish that failed can still have left the bytes there: off + // Unix the chunk is created under its final name, and if the write + // or the flush then fails, the cleanup that removes it can fail + // too. Releasing the reservation would hand back a charge for a + // file that is on the disk. + // + // The publish says so rather than this deciding from a later + // `is_file`. Asking the filesystem afterwards infers ownership from + // a name being occupied, which is true under the store lock and the + // shard lane and not true against anything out of band, and this + // file spends a lot of its length arguing that a name is not + // evidence. A bit set by the code that created the file is. + if left_behind { + reservation.commit(); + } + return Err(error); + } + }; + // Placed, not yet durable. A failure from here on leaves the bytes on the + // disk: the chunk is rightly not reported as stored, because a copy that is + // not durable must not authorise deleting another, but the space is spent + // all the same. Dropping the reservation would hand that charge back and + // admit the next write against room that is already gone. // - // Only when the file store really does not have it. A write can fail - // because the file that is already there could not be read to check it, - // and calling that key legacy-only while the file index still names it - // puts it in both views, where it stays answerable and vetoes retirement - // for good. - // The file half failed and the legacy half did not, so the environment - // holds the only copy and the key really is legacy-only now. Promoted - // from the in-flight note to the key set, which is the one moment that - // promotion is warranted: both outcomes are known. - if let Some(ref l) = legacy { - l.announced_write_finished(address); - if dual_written && !self.files.is_indexed(address) { - l.only.write().insert(*address); + // Only for a chunk this call published. `Duplicate` means the file was + // already there and was charged by whoever wrote it, so charging it again + // here would count one file twice and shrink the store's idea of its own + // disk on every retry. + if let Err(e) = flush_publication(&final_path, &shard) { + if matches!(outcome, PutOutcome::New) { + reservation.commit(); } + return Err(e); } - return Err(e); + // Index inside the lane, and only after the rename has returned. A + // concurrent delete of the same address therefore cannot interleave + // between publishing the file and admitting the key. + // + // Only for a chunk this call actually published. `Duplicate` says a file + // already wears the name, and a name is not evidence about the bytes under + // it: the four-way answer that decides whether they are good, wrong, absent + // or unreadable runs after the await below, and a caller whose future is + // dropped never reaches it. Admitting the key here would leave the node + // claiming, advertising and committing to bytes nothing has read, with no + // suspect or known-wrong mark to hold it back, and the sharpest case is a + // name the startup scan deliberately refused because what wears it is a + // fifo, a socket or a directory. The duplicate arm admits the key itself, + // once a read has proven the bytes. + if matches!(outcome, PutOutcome::New) { + index.write().insert(key); + // With the marks that would otherwise hold the key back. These bytes + // were hashed against their own name on the way in, so an older + // instance proven wrong or merely unreadable has just been replaced by + // a good one. Cleared here rather than after the await for the same + // reason the insert is here: a cancelled caller would leave the key + // indexed and suppressed at once, so a chunk this node really does hold + // would stay hidden from `exists` and `all_keys` until some later read + // happened to settle it. + known_wrong.write().remove(&key); + suspect.write().remove(&key); + // Settled here, inside the work, so a dropped awaiter cannot strand it. + reservation.commit(); + } + Ok(outcome) + }) + .await + .map_err(|e| Error::Storage(format!("Chunk store put task failed: {e}")))??; + + match outcome { + PutOutcome::Duplicate => self.settle_duplicate(address, content).await, + PutOutcome::New => { + // Freshly published bytes that were checked against their own name on the + // way in. The marks were already cleared inside the work, where a dropped + // caller cannot skip them; what is left here is only what a caller who is + // still waiting should see. + let mut stats = self.stats.write(); + stats.chunks_stored = stats.chunks_stored.saturating_add(1); + stats.bytes_stored = stats.bytes_stored.saturating_add(len); + drop(stats); + debug!("Stored chunk {} ({len} bytes)", hex::encode(address)); + Ok(true) } - }; - - // The file store has it, so it is not legacy-only, whether it was already there - // or this call put it there. The in-flight note goes at the same time: both - // writes have returned, so there is nothing left in flight to protect. - if let Some(ref l) = legacy { - l.only.write().remove(address); - l.announced_write_finished(address); - } - if already_in_legacy { - // Migrated for free: a hot key the copier no longer has to move. - return Ok(false); } - Ok(stored_in_files) } - /// Retrieve a chunk, verifying it against its address when configured to. + /// Decide what a name that was already taken actually means. + /// + /// Split out of [`Self::put`] because it is a different question. `put` puts bytes on + /// a disk; this reads bytes back to find out whether the ones already there are the + /// ones the caller is offering, which is the only thing that makes a duplicate safe to + /// report as stored. /// /// # Errors /// - /// Returns [`Error::Storage`] on an I/O failure, or when verification fails and no - /// intact copy is available. - pub async fn get(&self, address: &XorName) -> Result>> { - // Held for the whole read. A verifying read that finds rotted bytes throws the - // file away, and until this key is back in the legacy-only set there is a moment - // when it appears to live in neither store. Retirement waits behind this rather - // than deleting the copy the read is about to fall back on. - let _reading = self.retirement.read().await; - let fallback = self.legacy(); - match self.files.get(address).await { - Ok(Some(content)) => Ok(Some(content)), - Ok(None) => { - // The file store missed. If the legacy store answers, the key has to go - // back into the union view: the file index has just dropped it, and a key - // in neither view is skipped by the verification pass and destroyed by - // retirement. - self.serve_from_legacy(address, fallback).await + /// Returns [`Error::Storage`] when the existing file is absent or could not be read, + /// both of which mean this node must not report the chunk as held. + async fn settle_duplicate(&self, address: &XorName, content: &[u8]) -> Result { + // The file was already on disk, and its name is not evidence its contents + // are right. The startup scan indexes by name without reading anything, + // and on Windows a crash mid-write leaves a partial file under a real + // chunk name. Trusting the name here would acknowledge a chunk that was + // never stored, and then discard the good copy arriving to repair it. + // Every answer handled, because three of the four must not report the + // chunk as stored. A caller that hears success acts on it: a client drops + // its own copy, replication marks the key held, and the copier takes it + // out of the legacy-only set. + match self.stored_bytes_match(address).await { + StoredBytes::Good => { + // Admitted here, which is the first moment the bytes behind the + // name have been read and shown to hash to it. Idempotent: the + // ordinary case is a key the startup scan already indexed. + self.index.write().insert(*address); + { + let mut stats = self.stats.write(); + stats.duplicates = stats.duplicates.saturating_add(1); + } + Ok(false) } - Err(e) => { - // Whatever went wrong with the file, the legacy environment may still - // have the bytes, and while it is there it is the point of the bridge to - // use them. A verification failure means the file was thrown away; every - // other error (a full descriptor table, an I/O fault, an oversized file) - // leaves the file in place and unreadable. Both are unservable from the - // file store, and both are worth asking the other store about. - let verification_failed = format!("{e}").contains("verification failed"); + StoredBytes::Wrong => { warn!( - "Chunk {} could not be served from the file store ({e}); looking for a \ - copy in the legacy environment", + "Chunk {} was already on disk but its contents are wrong; \ + replacing it with the copy just offered", hex::encode(address) ); - let from_legacy = self.serve_from_legacy(address, fallback).await; - match from_legacy { - Ok(Some(content)) => Ok(Some(content)), - // Nothing anywhere. Report the original failure rather than a plain - // miss, so the caller can tell the difference. The key is only - // re-queued for copying when the file really went: an unreadable file - // that is still there is not legacy-only, and calling it so is how a - // key ends up claimed through one view and servable through neither. - Ok(None) => Err(e), - Err(legacy_error) => { - if verification_failed { - Err(e) - } else { - Err(legacy_error) - } - } - } + self.repair_holding_the_lane(address, content) + .await + .map(|()| true) } - } - } - - /// Serve a key the file store could not, from the legacy store, and put it back on - /// the copier's list. - /// - /// The whole sequence runs under the key's critical section, including the legacy - /// read. Reading first and locking afterwards would let a concurrent delete remove - /// both backings in between, and the key would then be re-inserted from bytes that no - /// longer exist anywhere: a phantom entry that `exists` reports and `get` never - /// satisfies. - async fn serve_from_legacy( - &self, - address: &XorName, - legacy: Option, - ) -> Result>> { - // The handle the caller took before it read the file. Not re-fetched here: the - // point of taking it early is that it has been held continuously since before the - // file could be thrown away, so retirement cannot have run in between. - let Some(legacy) = legacy else { - return Ok(None); - }; - let _lane = self.key_lock(address).await; - let Some(content) = legacy.lmdb.get(address).await? else { - return Ok(None); - }; - // Only if the file really is gone: a concurrent write or repair may have put a - // good one back while this was waiting for the lock. - if !self.files.is_indexed(address) { - legacy.only.write().insert(*address); - debug!( - "Chunk {} served from the legacy environment and re-queued for copying", + // The name was taken a moment ago and is not now, or was never a + // readable chunk file. Either way nothing holds these bytes, so say so + // rather than reporting a chunk that is not there. + StoredBytes::Absent => Err(Error::Storage(format!( + "Chunk {} was reported already on disk but nothing is there. Not \ + reporting it as stored.", hex::encode(address) - ); + ))), + // Replacing on an unanswered question would destroy a healthy copy, + // and reporting success would discard the offered one. The index entry + // stays: the file is still there, and dropping the entry would leave + // the chunk in neither this store's view nor the legacy one, which is + // what retirement destroys. Removing an entry is the quarantine path's + // job, and it removes the file with it, after a read that succeeded + // and proved the bytes wrong. + StoredBytes::Unreadable => Err(Error::Storage(format!( + "Chunk {} is on disk but could not be read to check it. Not \ + replacing it, and not reporting it as stored.", + hex::encode(address) + ))), } - Ok(Some(content)) } - /// Retrieve raw chunk bytes without content-address verification. - /// - /// # Errors + /// Take the critical section for one key. /// - /// Returns [`Error::Storage`] on an I/O failure. - pub async fn get_raw(&self, address: &XorName) -> Result>> { - // For the reason given on `get`: retirement must not run in the gap between the - // file going missing and this key being put back in the union view. - let _reading = self.retirement.read().await; - let fallback = self.legacy(); - let from_files = self.files.get_raw(address).await; - match from_files { - Ok(Some(content)) => return Ok(Some(content)), - Ok(None) => {} - // Same rule as `get`: while the legacy environment is there it may have the - // bytes, and this is the read that drives digest audits, possession checks - // and pruning. Answering "no digest" for a chunk the node can still produce - // is a failed audit for nothing. - Err(e) => { - let Some(legacy) = fallback else { - return Err(e); - }; - let _lane = self.key_lock(address).await; - return match legacy.lmdb.get_raw(address).await { - Ok(Some(content)) => Ok(Some(content)), - // Nothing anywhere: report the original failure, not a plain miss. - Ok(None) | Err(_) => Err(e), - }; - } - } - // Deliberately not gated on the legacy-only set. A chunk that was copied and then - // lost its file is not in that set, and the legacy environment is exactly where - // its bytes still are. An LMDB miss is cheap. Goes through the same path as - // `get`, so the key is restored to the union view rather than being served once - // and then quietly retired away. - let Some(legacy) = fallback else { - return Ok(None); - }; - let _lane = self.key_lock(address).await; - let raw = legacy.lmdb.get_raw(address).await?; - let missing_locally = !self.files.is_indexed(address); - if raw.is_some() && missing_locally { - legacy.only.write().insert(*address); + /// Held across await points, unlike the write lanes, so a whole logical transition for + /// a key excludes another. `None` only if the table were empty, which it is not. + async fn key_lock(&self, address: &XorName) -> Option> { + let lane = address.last().copied().unwrap_or(0) as usize; + match self.key_locks.get(lane) { + Some(lock) => Some(lock.lock().await), + None => None, } - Ok(raw) } - /// Check whether a chunk is stored, in either backing. - /// - /// An in-memory lookup: no syscall, no I/O, in both phases. - /// - /// # Errors + /// The address content hashes to. /// - /// Never fails. The signature is kept because callers treat an error as "absent". - pub fn exists(&self, address: &XorName) -> Result { - // Legacy first, deliberately. The copier writes the file and only then drops the - // key from the legacy-only set, so a reader that checked files first could - // observe the gap between those two steps and report a chunk the node definitely - // holds as absent. In this order the same interleaving yields a harmless - // duplicate instead. - if self - .legacy() - .is_some_and(|l| l.only.read().contains(address)) - { - return Ok(true); - } - self.files.exists(address) + /// A convenience the old facade offered and callers still use, so it stays with the + /// store rather than making every one of them reach for the client module. + #[must_use] + pub fn compute_address(content: &[u8]) -> XorName { + crate::client::compute_address(content) } - /// Does this node already hold `address` with exactly these bytes, and if it holds a - /// damaged copy, replace it with these? - /// - /// The question a responder has to answer before turning away an offered copy. Plain - /// [`Self::exists`] answers from names alone, and a name can outlive the bytes under - /// it: off Unix a chunk is created under its final name before it is written, so a - /// crash leaves a short file that `exists` reports as a chunk, and bit rot leaves a - /// full-length one. Acknowledging a client on the strength of either throws away the - /// copy that would repair it, and nothing offers it again. + /// Does this store already hold exactly these bytes under this address? /// - /// So this reads. It is affordable because the only caller is the client-facing PUT - /// path, reached when a client offers a chunk this node already has, and because the - /// alternative is keeping a chunk this node cannot serve and being penalised for it at - /// the next audit. + /// Asked by the protocol handler before it accepts a client's PUT, so the answer has + /// to be about the bytes and not about the name. A name on disk is not evidence: the + /// startup scan indexes by name without reading anything, and a partial file can wear a + /// real chunk name. Answering yes on a name would acknowledge a chunk that was never + /// stored and then discard the good copy that had just arrived to replace it. /// - /// `content` must already hash to `address`; the caller checks that before this is - /// reached, and a repair from bytes that do not would be worse than the damage. - /// - /// # Errors - /// - /// Never fails. An unreadable chunk answers `false`, so the offered copy is stored - /// through the ordinary path rather than refused. + /// A chunk that is on disk with the wrong bytes is repaired from what the caller + /// offered rather than turned away, because the caller has already checked those bytes + /// against the address. A chunk that cannot be read this time is not claimed as held + /// and not replaced either: the offer goes through the ordinary write path instead, + /// which never truncates a healthy file on the strength of an unanswered question. pub async fn holds_verified(&self, address: &XorName, content: &[u8]) -> bool { - // Held for the whole check, like every other operation that can reach the legacy - // environment. - let _using_legacy = self.retirement.read().await; - // And the key's own critical section, for the whole of it. Without it the pruner - // can delete both backings between the read and the answer, and the offered copy - // would be turned away for a chunk the node no longer has at all. + // The key's critical section for the whole check, so the answer is a linearizable + // statement about the store: at the moment this returns, the chunk was held and its + // bytes were these. + // + // It does not follow the answer out to the caller. The handler turns a `true` into + // an `AlreadyExists` and sends it afterwards, outside this lock, so a prune landing + // in between still means a peer is told to drop a copy of a chunk this node no + // longer has. Closing that would mean holding a per-key lock across a network + // response, which trades a narrow window for a much worse one. Replication finds + // the key missing and re-offers it. let _lane = self.key_lock(address).await; - if let Some(legacy) = self.legacy() { - if legacy.only.read().contains(address) { - // Held only in the legacy environment. Not taken on trust either: the - // bytes in there can be wrong too, and the copier drops such a key from - // the union when it finds out, which would leave no copy anywhere if this - // had turned the good one away. - if matches!(legacy.lmdb.get_raw(address).await, Ok(Some(bytes)) if bytes == content) - { - return true; - } - warn!( - "Chunk {} is in the legacy environment but its bytes are wrong; \ - storing the copy just offered instead", - hex::encode(address) - ); - if self.files.put(address, content).await.is_err() { - return false; - } - legacy.only.write().remove(address); - return true; - } - } - if !self.files.is_indexed(address) { + if !self.is_indexed(address) { return false; } // No cheap length pre-check. `metadata` failing is not the same as a length that // does not match, and off Unix replacing a chunk truncates it in place, so acting // on an unanswered question would empty a healthy sole copy. The read below // distinguishes them. - match self.files.get_raw(address).await { - // Byte-for-byte what the caller has, and the caller checked those bytes - // against the address before getting here. Nothing is wrong with this file. + match self.get_raw(address).await { + // Byte-for-byte what the caller has, and the caller checked those bytes against + // the address before getting here. Nothing is wrong with this file. Ok(Some(stored)) if stored == content => { - self.files.note_bytes_proven_good(address); + self.note_bytes_proven_good(address); true } Ok(_) => { @@ -809,4030 +1102,2911 @@ impl ChunkStore { copy just offered", hex::encode(address) ); - // Recorded before the repair is attempted, not after it succeeds. A - // repair can fail for capacity or I/O, and a chunk proven wrong that goes - // on looking healthy leaves a cached pre-retirement pass covering it, - // which deletes the legacy copy the repair would have come from. - self.files.note_known_wrong(address); - self.files.repair(address, content).await.is_ok() + // Recorded before the repair is attempted, not after it succeeds. A repair + // can fail for capacity or I/O, and a chunk proven wrong that goes on + // looking healthy is one the node keeps answering for. + self.note_known_wrong(address); + self.repair_holding_the_lane(address, content).await.is_ok() } // Unanswerable this time. Not claimed as held, so the offer goes through the // ordinary path, which writes it rather than replacing anything. + Err(_) => false, + } + } + + /// Decide what to do about a write of a chunk the index already names. + /// + /// `None` means the index was wrong and there is nothing on disk, so the caller + /// publishes it as new. Everything else is the answer. + async fn settle_indexed_duplicate( + &self, + address: &XorName, + content: &[u8], + ) -> Option> { + match self.stored_bytes_match(address).await { + StoredBytes::Good => { + trace!("Chunk {} already exists", hex::encode(address)); + { + let mut stats = self.stats.write(); + stats.duplicates = stats.duplicates.saturating_add(1); + } + Some(Ok(false)) + } + StoredBytes::Wrong => { + warn!( + "Chunk {} is indexed but its bytes are wrong; replacing it with the \ + copy just offered", + hex::encode(address) + ); + Some( + self.repair_holding_the_lane(address, content) + .await + .map(|()| true), + ) + } + // Indexed but gone: publish it fresh rather than replacing something that is + // not there. + StoredBytes::Absent => None, + // Unanswerable this time. Do not touch what is there, and do not tell the + // caller the chunk is safely stored either: a client would take that as an + // acknowledgement and drop the only other copy. The index entry stays, for + // the reason given on the same case after publication. + StoredBytes::Unreadable => Some(Err(Error::Storage(format!( + "Chunk {} is indexed but could not be read to check it. Not replacing it, \ + and not reporting it as stored.", + hex::encode(address) + )))), + } + } + + /// Whether the file already stored under `address` really hashes to it. + async fn stored_bytes_match(&self, address: &XorName) -> StoredBytes { + match self.get_raw(address).await { + Ok(Some(bytes)) if crate::client::compute_address(&bytes) == *address => { + // A read that hashed. It settles both questions. + self.clear_suspect(address); + self.clear_known_wrong(address); + StoredBytes::Good + } + Ok(Some(_)) => { + self.mark_known_wrong(address); + StoredBytes::Wrong + } + Ok(None) => StoredBytes::Absent, + // NOT the same as wrong. A file that could not be read this once may be + // perfectly good, and off Unix replacing it means opening it with `truncate`, + // which would destroy a healthy sole copy on the strength of a transient + // fault. Say so and let the caller leave it alone. Err(e) => { - warn!("Could not read {} to check it: {e}", hex::encode(address)); - false + debug!("Could not read {} to check it: {e}", hex::encode(address)); + self.mark_suspect(address); + StoredBytes::Unreadable } } } - /// Delete a chunk from both backings. + /// Replace the file behind an address with bytes that hash to it. + /// + /// Unlike [`Self::put`], this deliberately publishes **over** an existing name, for + /// repairing a file whose bytes no longer hash to their own. Doing it as + /// delete-then-put would leave a window with no copy at all. /// - /// A logical delete has to reach the legacy environment too, or the union view would - /// resurrect the key on the next read. It frees no space there — only removing the - /// environment whole does that — but it keeps the two views honest. + /// **Only call this once a read has shown the existing bytes are wrong.** On Unix the + /// replacement is atomic and a failure leaves the old file untouched. Off Unix it is + /// not: there is no durable rename there, so the existing file is truncated and + /// rewritten in place, and a crash part-way leaves a mixture. That is tolerable when + /// the bytes being replaced were already known to be wrong, and is data loss when they + /// were not. Nothing enforces the precondition, which is why it is stated here. /// /// # Errors /// - /// Returns [`Error::Storage`] if a file exists but cannot be removed. - pub async fn delete(&self, address: &XorName) -> Result { - // Shared, like every other operation that touches the legacy environment. Without - // it, retirement takes the exclusive guard and then waits for the environment to - // go idle while deletes keep starting new work in it, and the wait never ends. - let _using_legacy = self.retirement.read().await; + /// Returns [`Error::Storage`] if `content` does not hash to `address`, or the write + /// fails. + pub async fn repair(&self, address: &XorName, content: &[u8]) -> Result<()> { let _lane = self.key_lock(address).await; - // Behind whatever is already writing this key, and only this key. A write's - // blocking half outlives the future that started it, so one landing after this - // would put back a chunk the node had decided to prune. - self.files.wait_for_write(address).await; - // Legacy first, and only then the in-memory views. The other order removes the - // key from `only` and then, if the legacy delete fails, leaves bytes that live - // solely in the legacy store and are invisible to `exists`, `all_keys` and the - // pre-retirement verification, so retirement would take the only copy. - let from_legacy = match self.legacy() { - Some(legacy) => { - // A write for this key that nobody waited for may still be queued behind - // this delete. Letting it land afterwards would resurrect the key: the - // next reconciliation finds it in the environment and puts it back on the - // copier's list, undoing a prune the node decided on. Waited out here, - // holding the lane, so the delete is genuinely last. - // - // Both halves. A write has an environment half and a file half, either of - // which can be the one still running, and draining only the first leaves - // the second free to publish the file after this has deleted it. - // - // The environment half is found through the journal, which only dual - // writes keep. The file half is asked of the file store directly, because - // the copier and the repair path also spawn file writes and neither goes - // near that journal: using it as a proxy for "is anything writing this - // key" was a scope assumption, not a fact. - if legacy.pending.read().contains_key(address) { - legacy.lmdb.wait_idle().await; - } - let deleted = legacy.lmdb.delete(address).await?; - let was_only = legacy.only.write().remove(address); - // Every announcement for this key, not one of them: the drain above waited - // out whatever was in flight and this delete is deliberately last. - legacy.pending.write().remove(address); - deleted || was_only - } - None => false, - }; - let from_files = self.files.delete(address).await?; - Ok(from_files || from_legacy) + self.repair_holding_the_lane(address, content).await } - /// Every stored key, in ascending order, across both backings. - /// - /// The order is a correctness requirement: the commitment builder truncates the - /// responsible subset with `take(cap)` *before* the Merkle tree sorts it, so an - /// unstable order would make the node's published commitment depend on iteration - /// luck rather than on what it holds. - /// - /// # Errors + /// The body of [`Self::repair`], for callers that already hold the key's lane. /// - /// Returns [`Error::Storage`] if the file index cannot be read. - pub async fn all_keys(&self) -> Result> { - // Legacy first, for the reason given on `exists`. `merge_sorted` drops the - // duplicate that the overlap produces. - let legacy_only: Vec = self - .legacy() - .map(|l| l.only.read().iter().copied().collect()) - .unwrap_or_default(); - let file_keys = self.files.all_keys().await?; - if legacy_only.is_empty() { - return Ok(file_keys); + /// Split because the lane is a `tokio::sync::Mutex` and so is not reentrant: every + /// internal caller reaches this while holding it, and taking it again would deadlock + /// the task on itself. + async fn repair_holding_the_lane(&self, address: &XorName, content: &[u8]) -> Result<()> { + // The same ceiling `put` enforces. Without it a repair can install bytes the read + // path will refuse for ever, which is a chunk that verifies as present and can + // never be served. + if content.len() > MAX_CHUNK_SIZE { + return Err(Error::Storage(format!( + "Refusing to repair {} with {} bytes, over the {MAX_CHUNK_SIZE} byte \ + maximum", + hex::encode(address), + content.len() + ))); } - Ok(merge_sorted(&file_keys, legacy_only.iter())) + let computed = crate::client::compute_address(content); + if computed != *address { + return Err(Error::Storage(format!( + "Refusing to repair {} with content that hashes to {}", + hex::encode(address), + hex::encode(computed) + ))); + } + // The replacement exists alongside the original until the rename, so the room for + // it has to be there first. Reserved rather than merely checked: a plain check + // passes against a cached measurement, so concurrent repairs and PUTs can each be + // admitted against the same headroom and cross the reserve together. + // Moved into the work below, so it is released when the write finishes rather + // than when its caller stops waiting. A caller that goes away otherwise frees + // room that the detached write is still about to consume. + let reservation = self.capacity.reserve(content.len() as u64)?; + + let shard = self.chunks_dir.join(shard_name(address)); + let final_path = shard.join(hex::encode(address)); + let temp_path = shard.join(self.next_temp_name()); + let payload = content.to_vec(); + let lanes = Arc::clone(&self.write_lanes); + let index = Arc::clone(&self.index); + let shards_present = Arc::clone(&self.shards_present); + let chunks_dir = self.chunks_dir.clone(); + let lane = shard_index(address); + let key = *address; + let in_flight = self.begin_write(address); + let lease = Arc::clone(&self.lock); + let capacity = Arc::clone(&self.capacity); + let suspect = Arc::clone(&self.suspect); + let known_wrong = Arc::clone(&self.known_wrong); + + self.blocking_tracker + .spawn_blocking(move || -> Result<()> { + let _in_flight = in_flight; + let _lease = lease; + let _reservation = reservation; + let _lane = lanes.get(lane).map(parking_lot::Mutex::lock); + ensure_shard_dir(&chunks_dir, &shard, lane, &shards_present)?; + write_and_replace(&temp_path, &final_path, &payload, &shard)?; + index.write().insert(key); + // Settled here rather than after the await. The replacement has landed + // and hashes to its own name, so nothing is wrong with this chunk any + // more; a caller that stopped waiting would otherwise leave a healthy + // file excluded from everything the node claims to hold, and the + // measurement believing the store is a chunk smaller than it is. + suspect.write().remove(&key); + known_wrong.write().remove(&key); + capacity.invalidate(); + Ok(()) + }) + .await + .map_err(|e| Error::Storage(format!("Chunk store repair task failed: {e}")))??; + + // Everything the success means was recorded by the work that succeeded: the + // reservation released, the marks cleared, the measurement thrown away. Released + // rather than committed because a repair is not a new chunk, and the measurement + // discarded rather than adjusted because the file it replaced may have been + // shorter, which is exactly the case a repair fixes. + debug!("Repaired chunk {}", hex::encode(address)); + Ok(()) } - /// The keys the commitment builder should commit to. - /// - /// While the node is still bridging this is the whole union, because it can still - /// serve all of it and dropping the claim early would collapse its commitment (and - /// with it its quoted price) for no reason. Once it has settled on what it will keep, - /// this narrows to the file-backed set, which is exactly the point at which the node - /// stops claiming keys it is about to give up. + /// Retrieve a chunk, verifying it against its address when configured to. /// /// # Errors /// - /// Returns [`Error::Storage`] if the file index cannot be read. - pub async fn committable_keys(&self) -> Result> { - match self.migration_phase() { - MigrationPhase::Bridging => self.all_keys().await, - MigrationPhase::Committed | MigrationPhase::FilesOnly => self.files.all_keys().await, + /// Returns [`Error::Storage`] on an I/O failure, or when verification fails. A + /// chunk whose bytes do not hash to its name is removed and dropped from the index + /// before the error is returned, so it leaves `all_keys()` and ordinary replication + /// repairs it. + pub async fn get(&self, address: &XorName) -> Result>> { + let Some(content) = self.read_file(address).await? else { + trace!("Chunk {} not found", hex::encode(address)); + return Ok(None); + }; + + if self.config.verify_on_read { + let computed = crate::client::compute_address(&content); + if computed != *address { + { + let mut stats = self.stats.write(); + stats.verification_failures = stats.verification_failures.saturating_add(1); + } + warn!( + "Chunk verification failed: expected {}, computed {}", + hex::encode(address), + hex::encode(computed) + ); + // Said before it is acted on. Removing the file can fail or be cancelled, + // and a chunk proven wrong that goes on looking healthy is one the node + // keeps committing to and keeps being audited for. + self.mark_known_wrong(address); + self.quarantine_corrupt(address).await; + return Err(Error::Storage(format!( + "Chunk verification failed for {}", + hex::encode(address) + ))); + } + } + + if self.config.verify_on_read { + // The bytes hashed to their name. Whatever this store thought was wrong with + // them is not wrong with them, and a mark that outlives the fault it + // describes means the node can serve a chunk it will not claim, commit or + // offer. + self.clear_known_wrong(address); } + + let len = content.len() as u64; + { + let mut stats = self.stats.write(); + stats.chunks_retrieved = stats.chunks_retrieved.saturating_add(1); + stats.bytes_retrieved = stats.bytes_retrieved.saturating_add(len); + } + debug!("Retrieved chunk {} ({len} bytes)", hex::encode(address)); + Ok(Some(content)) } - /// Number of chunks currently stored, counted across both backings without - /// double-counting a chunk that is in each. + /// Retrieve raw chunk bytes without content-address verification. /// /// # Errors /// - /// Never fails. - pub fn current_chunks(&self) -> Result { - let (files, legacy) = self.split_counts(); - Ok(files.saturating_add(legacy)) + /// Returns [`Error::Storage`] on an I/O failure. + pub async fn get_raw(&self, address: &XorName) -> Result>> { + self.read_file(address).await } - /// Operation statistics. + /// Check whether a chunk is stored. /// - /// The cumulative counters are the file store's; `current_chunks` is the union. - #[must_use] - pub fn stats(&self) -> StorageStats { - let mut stats = self.files.stats(); - stats.current_chunks = self.current_chunks().unwrap_or(0); - stats + /// An in-memory lookup: no syscall, no I/O. + /// + /// # Errors + /// + /// Never fails. The signature keeps the shape the LMDB store had, because callers + /// treat the error as "assume absent". + pub fn exists(&self, address: &XorName) -> Result { + if self.is_unservable(address) { + return Ok(false); + } + Ok(self.is_indexed(address)) } - /// Compute a content address (BLAKE3 hash). + /// Is this chunk one the node must not answer for? #[must_use] - pub fn compute_address(content: &[u8]) -> XorName { - crate::client::compute_address(content) + fn is_unservable(&self, address: &XorName) -> bool { + self.suspect.read().contains(address) || self.known_wrong.read().contains(address) } - /// The node root directory. + /// Is this chunk in the index, whether or not it can currently be read? + /// + /// The physical question, as against [`Self::exists`]'s question about what the node + /// is willing to claim. The migration must ask this one: a suspect chunk is still a + /// file this store has, and treating it as absent would put the key in the legacy-only + /// set, from where the union view advertises it again — a key the node claims through + /// one view and cannot serve through either. #[must_use] - pub fn root_dir(&self) -> &Path { - &self.config.root_dir + pub fn is_indexed(&self, address: &XorName) -> bool { + self.index.read().contains(address) } - /// Reject work early when the disk cannot take another chunk at all. + /// Delete a chunk, returning whether it was present. + /// + /// `unlink` returns the blocks to the filesystem immediately. That is the whole + /// point of this store: no free list, no compaction, no free space required to + /// reclaim space. /// /// # Errors /// - /// Returns [`Error::Storage`] when free space is below the configured reserve. - pub fn check_capacity(&self) -> Result<()> { - self.files.check_capacity() + /// Returns [`Error::Storage`] if the file exists but cannot be removed. The index + /// keeps the key in that case, because the bytes are still on disk. + pub async fn delete(&self, address: &XorName) -> Result { + // The key's whole critical section, held across the wait below and the unlink. + let _lane = self.key_lock(address).await; + // Behind whatever is already writing this key, and only this key. A write's + // blocking half outlives the future that started it, so one landing after this + // would put back a chunk the node had decided to prune, and the next thing to look + // would find it in a store that no longer claims it. + self.wait_for_write(address).await; + + let path = self.chunk_path(address); + let lanes = Arc::clone(&self.write_lanes); + let index = Arc::clone(&self.index); + let lane = shard_index(address); + let key = *address; + // Carried into the closure for the reason `put`, `repair` and the startup scan + // carry it: this work outlives the future that started it, so a cancelled caller + // that drops the last `ChunkStore` would otherwise release the directory to another + // process while an unlink is still queued against it. Deleting is the operation + // where that matters most. + let lease = Arc::clone(&self.lock); + + let (existed, freed) = self + .blocking_tracker + .spawn_blocking(move || -> Result<(bool, u64)> { + let _lease = lease; + let _lane = lanes.get(lane).map(parking_lot::Mutex::lock); + let len = std::fs::metadata(&path).map_or(0, |m| m.len()); + let removed = match std::fs::remove_file(&path) { + Ok(()) => { + // Without this a crash can resurrect the entry on ext4, XFS, + // btrfs and APFS: the unlink is in the page cache, the directory + // is not. + if let Some(shard) = path.parent() { + fsync_dir_best_effort(shard); + } + true + } + // Already gone: the index was stale. Still a successful delete as + // far as the caller is concerned. + Err(e) if e.kind() == ErrorKind::NotFound => false, + Err(e) => { + return Err(Error::Storage(format!( + "Failed to delete chunk file {}: {e}", + path.display() + ))) + } + }; + // Index only after the filesystem operation has succeeded. On the error + // path above the entry stays, because the bytes are still on disk. + let was_indexed = index.write().remove(&key); + Ok((removed || was_indexed, if removed { len } else { 0 })) + }) + .await + .map_err(|e| Error::Storage(format!("Chunk store delete task failed: {e}")))??; + + if freed > 0 { + self.capacity.record_removed(freed); + debug!("Deleted chunk {}", hex::encode(address)); + } + Ok(existed) } - /// Whether the store can take a write at all right now. + /// Return every stored key, in ascending order. /// - /// Answered by the file store, which is where writes land. The legacy environment's - /// own verdict is deliberately not consulted: it accounts for pages it can reuse - /// internally, and a reusable page in a store this node is moving *off* says nothing - /// about whether the file it is about to write will fit. + /// The order is a correctness requirement, not a convenience: the commitment + /// builder truncates the responsible subset with `take(cap)` before the Merkle tree + /// sorts it, so an unstable order would make the node's published commitment depend + /// on iteration luck. /// - /// Three-way, not two. The verification cycle treats `Full` as a standing condition - /// worth minutes of backoff, so folding a failed free-space query into it would latch - /// a transient filesystem hiccup into a stall on a node that is not full at all. - #[must_use] - pub(crate) fn capacity_verdict(&self) -> crate::storage::CapacityVerdict { - self.files.capacity_verdict() + /// # Errors + /// + /// Never fails. The signature matches the LMDB store's. + // Async without awaiting anything, deliberately: the whole point of this store is + // that the key set is already in memory. Callers are spread across the replication + // engine and cannot all be de-async'd in this change. + // + // Two lint names because they were renamed between toolchains, and `unknown_lints` + // so whichever one the compiler in use has never heard of stays quiet. + #[allow(unknown_lints)] + #[allow(clippy::unused_async, clippy::unused_async_trait_impl)] + pub async fn all_keys(&self) -> Result> { + // Copied out first so neither lock is held while the other is taken, and so the + // usual case, where nothing is suspect, costs one clone of an empty set. + let mut unservable: HashSet = self.suspect.read().clone(); + unservable.extend(self.known_wrong.read().iter().copied()); + let keys = self.index.read().clone(); + if unservable.is_empty() { + return Ok(keys.into_iter().collect()); + } + Ok(keys + .into_iter() + .filter(|key| !unservable.contains(key)) + .collect()) } - /// Reject work early when the disk cannot take `bytes` more. + /// Stop answering for a chunk this store could not read. /// - /// Free bytes alone stopped being a sufficient answer once chunks became files. + /// The file stays. It may be perfectly good and unreadable only for the moment, and + /// deleting it, or dropping it from the index, is how a chunk ends up in neither this + /// store's view nor the legacy one, which is what retirement destroys. /// - /// # Errors + /// What does change is what the node says about it. A chunk it cannot read is one it + /// cannot serve, and claiming it anyway puts the key in signed commitments, answers + /// presence probes with a yes, suppresses the replication that would repair it, and + /// earns a penalty at the next commitment-bound audit. Those penalties are not + /// suspended. + fn mark_suspect(&self, address: &XorName) { + if self.suspect.write().insert(*address) { + warn!( + "Chunk {} is on disk but could not be read; this node stops answering for \ + it until a read succeeds", + hex::encode(address) + ); + } + } + + /// Stop answering for a chunk a read has proven wrong. /// - /// Returns [`Error::Storage`] when the write would not fit above the reserve. - pub fn check_capacity_for(&self, bytes: u64) -> Result<()> { - self.files.check_capacity_for(bytes) + /// Unlike a chunk that merely could not be read, a later read does not clear this. + /// The bytes are wrong, and reading them again says the same thing; only replacing + /// them or removing them settles it. + /// + /// For callers outside this module that have proven it themselves. + pub fn note_known_wrong(&self, address: &XorName) { + self.mark_known_wrong(address); } - /// Wait until every blocking task in either backing has finished. - pub async fn wait_idle(&self) { - self.files.wait_idle().await; - if let Some(legacy) = self.legacy() { - legacy.lmdb.wait_idle().await; + /// Stop answering for a chunk a read has proven wrong. + fn mark_known_wrong(&self, address: &XorName) { + if self.known_wrong.write().insert(*address) { + warn!( + "Chunk {} does not match its name; this node stops answering for it until \ + it is repaired or removed", + hex::encode(address) + ); } } - // ── Migration ─────────────────────────────────────────────────────────── - - /// Where the node is in the migration. - #[must_use] - pub fn migration_phase(&self) -> MigrationPhase { - self.state.read().phase + /// A caller outside this module has proven the stored bytes are right. + pub fn note_bytes_proven_good(&self, address: &XorName) { + self.clear_known_wrong(address); + self.clear_suspect(address); } - /// A snapshot of the persisted migration marker. - #[must_use] - pub fn migration_state(&self) -> MigrationState { - self.state.read().clone() + /// Answer for a chunk again, after it has been replaced or removed. + fn clear_known_wrong(&self, address: &XorName) { + self.known_wrong.write().remove(address); } - /// The migration settings this store was built with. - #[must_use] - pub fn migration_config(&self) -> &MigrationConfig { - &self.config.migration + /// Answer for a chunk again, after a read that worked. + fn clear_suspect(&self, address: &XorName) { + if !self.suspect.read().contains(address) { + return; + } + if self.suspect.write().remove(address) { + info!( + "Chunk {} could be read again; this node answers for it once more", + hex::encode(address) + ); + } } - /// Whether a legacy environment is still open. - #[must_use] - pub fn has_legacy(&self) -> bool { - self.legacy.read().is_some() + /// Number of chunks currently stored. + /// + /// The physical count: every name in the index, including chunks the node has stopped + /// answering for because a read found them wrong or could not read them at all. It is + /// deliberately not the same number as `all_keys().len()`, which is what the node is + /// willing to claim and so leaves those out. + /// + /// Anything asking "how much is on this disk" wants this one, and that is what its + /// callers ask: the migration's progress, the storage stats, and the size an audit is + /// built for. Anything asking "what will this node answer for" wants `all_keys`. + /// Quietly filtering this one would move all three of those without saying so, which + /// is why the difference is written down here rather than removed. + /// + /// # Errors + /// + /// Never fails. The signature matches the LMDB store's. + pub fn current_chunks(&self) -> Result { + Ok(self.index.read().len() as u64) } - /// The keys the legacy environment still holds alone, ascending. + /// Operation statistics, with the live chunk count filled in. #[must_use] - pub fn legacy_only_keys(&self) -> Vec { - self.legacy() - .map(|l| l.only.read().iter().copied().collect()) - .unwrap_or_default() + pub fn stats(&self) -> StorageStats { + let mut stats = self.stats.read().clone(); + stats.current_chunks = self.index.read().len() as u64; + stats } - /// Bytes the legacy environment occupies, as the filesystem sees it. + /// The node root directory this store was configured with. #[must_use] - pub fn legacy_bytes(&self) -> u64 { - std::fs::metadata(self.legacy_env_dir.join(LEGACY_DATA_FILE)).map_or(0, |m| m.len()) + pub fn root_dir(&self) -> &Path { + &self.config.root_dir } - /// Test-only handle to the file store's put gate. - #[cfg(any(test, feature = "test-utils"))] + /// The directory holding the shard tree. #[must_use] - pub fn test_put_gate(&self) -> Arc> { - self.files.test_put_gate() + pub fn chunks_dir(&self) -> &Path { + &self.chunks_dir } - /// Test-only: adjust the persisted migration marker directly. + /// Reject work early when the disk cannot take another chunk at all. /// - /// Real transitions go through [`Self::commit_to_files`] and - /// [`Self::note_commitment_rebuilt`]; this exists so a test can put a store into a - /// state that would otherwise take hours of wall clock to reach. - #[cfg(any(test, feature = "test-utils"))] - pub fn force_migration_state(&self, f: F) { - f(&mut self.state.write()); + /// # Errors + /// + /// Returns [`Error::Storage`] when free space is below the configured reserve. + pub fn check_capacity(&self) -> Result<()> { + self.capacity.check(0) } - /// Copy up to `keys.len()` chunks out of the legacy environment into files. + /// Three-way answer to "can this store take a write right now". /// - /// Stops as soon as free space would fall below `slack` above the configured - /// reserve, so a migration never fills the disk it is trying to free. + /// Kept distinct from [`Self::check_capacity`] because a failed free-space query and a + /// genuinely full disk are not the same thing, and the replication verification cycle + /// depends on the difference: a full disk is a standing condition worth minutes of + /// backoff, while a `statvfs` that failed says nothing about available space and may + /// well succeed on the next pass. + #[must_use] + pub fn capacity_verdict(&self) -> crate::storage::CapacityVerdict { + match self.capacity.measure_available() { + Some(available) if available < self.capacity.reserve => { + crate::storage::CapacityVerdict::Full + } + Some(_) => crate::storage::CapacityVerdict::Writable, + None => crate::storage::CapacityVerdict::Unknown, + } + } + + /// Reject work early when the disk cannot take `bytes` more. /// /// # Errors /// - /// Returns [`Error::Storage`] only for failures that are not per-key: a per-key - /// problem is counted in the report and the pass continues. - pub async fn copy_batch( - &self, - keys: &[XorName], - slack: u64, - throttle_mib_per_sec: u64, - shutdown: &CancellationToken, - ) -> Result { - let mut report = CopyReport::default(); - let Some(legacy) = self.legacy() else { - return Ok(report); - }; + /// Returns [`Error::Storage`] when the write would not fit above the reserve. + pub fn check_capacity_for(&self, bytes: u64) -> Result<()> { + self.capacity.check(bytes) + } - for key in keys { - // Checked per chunk, not per batch. Everything here is idempotent and - // re-derived at the next start, so stopping between two chunks costs nothing - // and stops shutdown waiting out a whole pass. - if shutdown.is_cancelled() { - break; - } - let lane = self.key_lock(key).await; - // Re-checked inside the critical section. A prune that landed while this - // pass was running has already taken the key out of the legacy-only set, and - // copying it now would resurrect a chunk the node deliberately deleted. - if !legacy.only.read().contains(key) { - continue; - } - // Physically, again: a chunk the store holds and cannot read is not one to - // copy over the top of, and it is not legacy-only either. - if self.files.is_indexed(key) { - legacy.only.write().remove(key); - continue; - } - // Reserve room for a full chunk plus the slack floor before reading, so the - // copier stops with headroom rather than on a failed write. - if self.files.check_capacity_for(slack).is_err() { - report.stopped_for_space = true; - break; - } + /// Test-only handle to the put gate. + /// + /// Hold the write half to park the next write inside its blocking closure, for + /// example to prove that shutdown waits for a write whose awaiter was dropped. + #[cfg(test)] + fn test_put_gate(&self) -> Arc> { + Arc::clone(&self.test_put_gate) + } - let Some(bytes) = legacy.lmdb.get_raw(key).await? else { - report.vanished += 1; - legacy.only.write().remove(key); - continue; - }; - let len = bytes.len() as u64; - - match self.files.put(key, &bytes).await { - Ok(_) => { - legacy.only.write().remove(key); - report.copied += 1; - report.bytes += len; - } - Err(e) => { - let message = format!("{e}"); - // Bigger than this build will ever serve. The legacy store took it - // through an API with no size bound; the file store will not, and no - // amount of retrying changes that. Counted as unusable and removed, - // like a record whose bytes do not match, or one such record would - // stop this node and every node sharing its disk from ever reclaiming - // space. - if message.contains("byte maximum") { - warn!( - "Chunk {} in the legacy environment is larger than this build \ - will store; removing it. It cannot be served either way.", - hex::encode(key) - ); - match legacy.lmdb.delete(key).await { - Ok(_) => { - legacy.only.write().remove(key); - report.unusable += 1; - } - Err(e) => warn!( - "Oversized chunk {} could not be removed from the legacy \ - environment: {e}. The environment stays.", - hex::encode(key) - ), - } - continue; - } - if message.contains("Content address mismatch") { - // The legacy bytes do not hash to their own key, so this chunk - // cannot be reproduced and was never servable. Stop advertising - // it rather than carrying a key we cannot answer for. - // - // Deleted from the environment too, and only dropped from the key - // set once that has worked. Leaving the record behind puts the - // key in neither view, which the pre-retirement pass reads as a - // chunk to protect and puts straight back — and the next copier - // pass drops it again. One malformed record would keep a node, - // and every node sharing its disk, from ever reclaiming space. - warn!( - "Chunk {} in the legacy environment does not match its address; \ - removing it so replication can repair it", - hex::encode(key) - ); - match legacy.lmdb.delete(key).await { - Ok(_) => { - legacy.only.write().remove(key); - report.unusable += 1; - } - Err(e) => warn!( - "Chunk {} does not match its address and could not be \ - removed from the legacy environment: {e}. It stays on the \ - list and the environment stays.", - hex::encode(key) - ), - } - continue; - } - if message.contains("Insufficient disk space") { - report.stopped_for_space = true; - break; - } - return Err(e); - } - } + /// Test-only handle to the gate that parks a put before it registers itself. + #[cfg(test)] + fn test_pre_registration_gate(&self) -> Arc> { + Arc::clone(&self.test_pre_registration_gate) + } - // Outside the critical section on purpose: at 32 MiB/s a 4 MiB chunk sleeps - // for over a tenth of a second, and a shard lane held for that would stall - // every write sharing its last address byte for the whole pass. - drop(lane); - if let Some(delay) = throttle_delay(len, throttle_mib_per_sec) { - tokio::time::sleep(delay).await; - } - } - Ok(report) + /// Test-only: how many puts have reached the pre-registration gate. + #[cfg(test)] + fn test_reached_pre_registration(&self) -> u64 { + self.test_reached_pre_registration + .load(std::sync::atomic::Ordering::Acquire) } - /// Settle on the file-backed set: from now on the node commits only to what it will - /// keep, while still serving everything it ever committed to. - /// - /// # Errors + /// Register a write of `address` and hand back the token that clears it. /// - /// Returns [`Error::Storage`] if the marker cannot be persisted. - pub fn commit_to_files(&self) -> Result<()> { - let shed = self - .legacy() - .map_or(0, |l| l.only.read().len().try_into().unwrap_or(u64::MAX)); - let kept = self.files.current_chunks().unwrap_or(0); - // Written to disk before it is published in memory. The other order leaves this - // process acting as `Committed` (and so committing only to file-backed keys) - // while the marker still says `Bridging`, so a restart would silently undo it. - let candidate = { - let state = self.state.read(); - if state.phase != MigrationPhase::Bridging { - return Ok(()); - } - MigrationState { - phase: MigrationPhase::Committed, - committed_at_unix: None, - rebuilds_since_commit: 0, - shed_key_count: shed, - kept_key_count: kept, - ..state.clone() - } - }; - candidate.save(&self.config.root_dir)?; - *self.state.write() = candidate; - if shed == 0 { - info!("Committed to the file-backed key set; nothing has to be shed"); - } else { - info!( - "Committed to the file-backed key set; {shed} chunk(s) will be shed and \ - refetched once the legacy environment is gone and there is room" - ); + /// The token must be moved into the blocking closure that does the work, so the entry + /// is cleared by the thread that finishes rather than by a caller that may be gone. + fn begin_write(&self, address: &XorName) -> WriteInFlight { + *self.writing.lock().entry(*address).or_insert(0) += 1; + WriteInFlight { + writing: Arc::clone(&self.writing), + finished: Arc::clone(&self.write_finished), + address: *address, } - Ok(()) } - /// Record that the commitment builder has read and published the committable set. + /// Wait until nothing is part-way through writing `address`. /// - /// The retirement gate counts these: one proves the builder saw the new set, two - /// prove it survived a rotation, which is what makes the answerability window - /// meaningful rather than notional. - pub fn note_commitment_rebuilt(&self) { - let should_save = { - let mut state = self.state.write(); - if state.phase != MigrationPhase::Committed { + /// For callers that must be last: a delete whose key still has a write in flight + /// would be undone by that write landing afterwards. + pub async fn wait_for_write(&self, address: &XorName) { + loop { + // Registered before the check, so a clear between the two is not missed. + let waiting = self.write_finished.notified(); + if !self.writing.lock().contains_key(address) { return; } - if state.committed_at_unix.is_none() { - state.committed_at_unix = Some(crate::storage::migration::now_unix()); - } - state.rebuilds_since_commit = state.rebuilds_since_commit.saturating_add(1); - state.rebuilds_since_commit <= REQUIRED_REBUILDS_BEFORE_RETIRE - }; - if should_save { - let snapshot = self.state.read().clone(); - if let Err(e) = snapshot.save(&self.config.root_dir) { - warn!("Could not persist the migration marker: {e}"); - } + waiting.await; } } - /// Whether every gate on deleting the legacy environment is satisfied. + /// How many blocking tasks this store currently has in flight. Tests only. /// - /// `still_answerable` is asked of each key the node is about to give up: the pruner's - /// existing retention contract, reused verbatim. A key still covered by a retained - /// commitment slot vetoes the delete, because the node could still be challenged on it. - pub fn retirement_blocker(&self, still_answerable: F) -> Option - where - F: Fn(&XorName) -> bool, - { - // Before anything else, and whether or not there is a handle. An environment this - // node cannot classify must not be retired, and asking only on the no-handle path - // meant the ordinary path never asked: a node holding its store open went through - // every gate, renamed the directory aside and deleted it. - if self.legacy_cannot_be_classified() { - return Some(format!( - "{} cannot be read well enough to say whether it was already retired. \ - Nothing will be deleted until it can. Check that the directory and \ - anything inside it can be read.", - self.legacy_env_dir.display() - )); - } - if !self.has_legacy() { - // No handle is not the same as no environment. A rename that failed and then - // could not be reopened leaves exactly that: the directory is still on disk - // and this node can no longer read it. Answering "nothing blocks retirement" - // would have the driver log the migration complete over a store that is still - // there and still holding chunks nothing else can serve. - let mark = retirement_mark(&self.legacy_env_dir); - if legacy_present(&self.config.root_dir).unwrap_or(true) && !mark.permits_removal() { - // Two different situations wearing one message would send an operator to - // the wrong place. One is a store this node cannot open; the other is a - // store nothing can even classify, which usually means a permission or a - // mount, and which the node deliberately will not act on either way. - if mark == RetirementMark::Unknown { - return Some(format!( - "{} is still on disk and this node cannot tell whether it was \ - retired, so it will neither open it nor remove it. Check that the \ - directory and anything inside it can be read.", - self.legacy_env_dir.display() - )); - } - return Some(format!( - "{} is still on disk but this node has no handle to it. It cannot be \ - read, verified or removed until the node is restarted.", - self.legacy_env_dir.display() - )); - } - return None; - } - if !self.config.migration.retire_legacy { - return Some( - "retirement is disabled in this release (storage.migration.retire_legacy)".into(), - ); - } - // A linked environment is never retired automatically. Retirement renames the - // path and then deletes what is behind it, and behind a link is a directory - // somewhere else that this node does not own. Copying still happens; only the - // removal is refused, so the node ends up serving from files with its old store - // intact and its operator told what to do about it. - if is_a_link(&self.legacy_env_dir) { - return Some(format!( - "{} is a link rather than a directory. The chunks are being copied out of \ - it, but it will not be deleted: what it points at is not this node's to \ - remove. Once the migration has settled, delete it by hand.", - self.legacy_env_dir.display() - )); - } - let state = self.state.read().clone(); - if state.phase != MigrationPhase::Committed { - return Some(format!("phase is {:?}, not Committed", state.phase)); - } - if state.rebuilds_since_commit < REQUIRED_REBUILDS_BEFORE_RETIRE { - return Some(format!( - "only {} of {REQUIRED_REBUILDS_BEFORE_RETIRE} commitment rebuilds observed", - state.rebuilds_since_commit - )); - } - if !state.retire_delay_elapsed(&self.config.migration) { - return Some(format!( - "the {}h retirement delay has not elapsed", - self.config.migration.effective_retire_delay_hours() - )); - } - if let Some(key) = self - .legacy_only_keys() - .into_iter() - .find(|k| still_answerable(k)) - { - return Some(format!( - "chunk {} is still answerable under a retained commitment", - hex::encode(key) - )); - } - None + /// Lets a test wait for work to have actually started rather than guessing at a + /// delay, which is the difference between a test that proves something and one that + /// passes because the machine was quick. + #[cfg(test)] + #[must_use] + pub(crate) fn tasks_in_flight(&self) -> usize { + self.blocking_tracker.len() } - /// Re-hash every chunk that both stores hold, repairing the file from the legacy - /// copy when they disagree. - /// - /// A filename is not proof the bytes behind it are good. The startup scan reads - /// names only, so a file that was truncated or that rotted while the node was down is - /// indexed, counted as copied, committed to, and would have its intact legacy copy - /// deleted underneath it. The first verified read would then find the corruption with - /// nothing left to repair from. This pass is what turns "a file with that name - /// exists" into "those bytes are that chunk", and it is why it runs before - /// retirement rather than after. - /// - /// # Errors + /// Wait until every blocking task this store spawned has finished. /// - /// Returns [`Error::Storage`] if the legacy key set cannot be read. - pub async fn verify_before_retire( - &self, - throttle_mib_per_sec: u64, - shutdown: &CancellationToken, - ) -> Result { - let mut report = VerifyReport::default(); - // Taken BEFORE anything is read. Stamping it at the end would absorb exactly the - // failures this exists to catch: a chunk verified early in the pass that stops - // being readable before the pass finishes would leave the report carrying the - // already-incremented count, and both later checks would see it match. - let health_at_start = self.files.health_generation(); - let Some(legacy) = self.legacy() else { - report.ran = true; - report.health = health_at_start; - return Ok(report); + /// Dropping the awaiting future does not cancel a `spawn_blocking` closure, so + /// shutdown has to wait for the closure itself. + pub async fn wait_idle(&self) { + self.blocking_tracker.close(); + self.blocking_tracker.wait().await; + self.blocking_tracker.reopen(); + } + + /// Absolute path of a chunk file. + fn chunk_path(&self, address: &XorName) -> PathBuf { + self.chunks_dir + .join(shard_name(address)) + .join(hex::encode(address)) + } + + /// A temp name unique to this store instance, and distinguishable from a chunk name. + /// + /// The nonce matters: two `ChunkStore`s on one root in one process share a PID, and a + /// recycled PID collides with an age-gated leftover. Either way `create_new` would + /// fail and surface as a spurious write error. + fn next_temp_name(&self) -> String { + let seq = self.temp_seq.fetch_add(1, Ordering::Relaxed); + format!( + "{TEMP_PREFIX}{}.{:08x}.{seq}", + std::process::id(), + self.nonce + ) + } + + /// Read a chunk file, dropping the index entry if the file has vanished. + async fn read_file(&self, address: &XorName) -> Result>> { + let path = self.chunk_path(address); + let read = self + .blocking_tracker + .spawn_blocking(move || -> Result>> { + match open_regular(&path) { + Ok(Some(f)) => read_bounded(f, &path).map(Some), + Ok(None) => Ok(None), + Err(e) => Err(e), + } + }) + .await + .map_err(|e| Error::Storage(format!("Chunk store read task failed: {e}")))?; + + // Every read decides the question, not only the ones that were checking. A read + // that failed means this chunk cannot be served, whoever asked; a read that + // worked means it can be, whoever asked. Doing this anywhere else leaves a key + // stuck unadvertised after the fault has cleared, or advertised after it has not. + let read = match read { + Ok(read) => { + self.clear_suspect(address); + read + } + Err(e) => { + self.mark_suspect(address); + return Err(e); + } }; - report.ran = true; - - // Names before bytes. A chunk whose contents are durable but whose directory entry - // is not is still lost to a power loss, and the legacy copy is about to be deleted - // on the strength of this proof. Any failure here is a proof this pass did not - // produce. - if let Err(e) = self.files.flush_namespace() { - report.unrepairable = report.unrepairable.saturating_add(1); + + if read.is_none() && self.forget_if_absent(address).await { + // The file went away underneath us. Stop advertising the key so the close + // group notices the shortfall and replication puts it back. warn!( - "Could not make the file store's directory entries durable: {e}. The legacy \ - environment stays until they are." + "Chunk {} is indexed but its file is missing; dropped from the index so \ + replication can repair it", + hex::encode(address) ); - return Ok(report); } - - let legacy_keys = legacy.lmdb.all_keys().await?; - let total = legacy_keys.len(); - info!("Verifying {total} chunk(s) before removing the legacy environment"); - let mut since_log = 0u64; - for key in legacy_keys { - // This pass is a full read of the store and can run for hours. A shutdown - // must not wait it out, and an incomplete pass is simply not a clean proof. - if shutdown.is_cancelled() { - report.unrepairable = report.unrepairable.saturating_add(1); - debug!("Pre-retirement verification stopped for shutdown"); - return Ok(report); - } - // Under the key's critical section, so the two questions below are asked of - // one moment. Without it a write can publish the file and take the key out of - // the legacy-only set in between, and this pass would put it straight back. - let classified = { - let _lane = self.key_lock(&key).await; - // The physical question. A chunk the store holds but cannot currently - // read is still one it holds, and calling it absent here would put the - // key in the legacy-only set, where the union view advertises it again. - let in_files = self.files.is_indexed(&key); - let legacy_only = legacy.only.read().contains(&key); - if in_files && legacy_only { - // In both views at once, which nothing else clears once the copier - // has stopped running. The file store has it, so the legacy-only set - // is the one that is wrong: an answerable key in that set vetoes - // retirement for as long as the process lives. - debug!( - "Chunk {} was in both views; the file store has it, so it is no \ - longer legacy-only", - hex::encode(key) - ); - legacy.only.write().remove(&key); - } - (in_files, legacy_only) - }; - if !classified.0 { - // Known to be legacy-only, which is what a key this node is giving up - // looks like. Whether it may go is the gates' decision, not this pass's. - if classified.1 { - continue; + Ok(read) + } + + /// Drop an index entry whose file is genuinely gone. + /// + /// Re-checks under the address's write lane, so a chunk republished between the + /// failing read and this call keeps its entry. + async fn forget_if_absent(&self, address: &XorName) -> bool { + // Not suspect any more: it is not unreadable, it is not there. + self.clear_suspect(address); + let path = self.chunk_path(address); + let lanes = Arc::clone(&self.write_lanes); + let index = Arc::clone(&self.index); + let lane = shard_index(address); + let key = *address; + // The bump happens inside the closure, with the mutation it describes. The + // closure runs to completion on its own thread whether or not anyone is still + // awaiting it, so bumping after the await is skipped entirely when a shutdown + // drops the caller — and the index change it was meant to announce still lands. + // A cached pre-retirement proof would then stay valid over a store that had + // quietly lost a chunk. + self.blocking_tracker + .spawn_blocking(move || { + let _lane = lanes.get(lane).map(parking_lot::Mutex::lock); + if path.exists() { + return false; } - // In neither view. However that came about — a publish that failed, a - // file quarantined for corruption, a name this store stopped advertising — - // the environment holds the only copy, and nothing is looking after it: - // the gates only ever see the legacy-only set. Put it back there and - // refuse the proof this pass. What neither view protects is exactly what - // retirement destroys. + let forgotten = index.write().remove(&key); + forgotten + }) + .await + .unwrap_or(false) + } + + /// Remove a chunk whose bytes do not match its name, and stop advertising it. + /// + /// Re-reads and re-verifies under the address's write lane first. A read that failed + /// verification is rare enough that paying for one extra read is worth never + /// discarding a chunk that a concurrent write had already repaired. + async fn quarantine_corrupt(&self, address: &XorName) { + let path = self.chunk_path(address); + let lanes = Arc::clone(&self.write_lanes); + let index = Arc::clone(&self.index); + let lane = shard_index(address); + let key = *address; + // For the reason given on `forget_if_absent`: this closure outlives its awaiter, + // and the change it makes has to be announced by the same thread that makes it. + // And the store-lock lease, for the reason `put`, `repair`, `delete` and the + // startup scan carry it: this closure outlives its awaiter, so without it a + // cancelled verification whose caller dropped the last `ChunkStore` would unlink + // inside a directory a second process had already been handed. + let lease = Arc::clone(&self.lock); + let outcome = + self.blocking_tracker + .spawn_blocking(move || -> std::io::Result { + let _lease = lease; + let _lane = lanes.get(lane).map(parking_lot::Mutex::lock); + // Nothing is thrown away without proof. A re-read that fails says the + // question could not be answered this time, not that the bytes are wrong, + // and a repair may have published a good copy since the read that brought + // us here. Treating either as corruption deletes a chunk this node has. + let buf = match open_regular(&path) { + Ok(Some(f)) => read_bounded(f, &path) + .map_err(|e| std::io::Error::other(e.to_string()))?, + Ok(None) => { + index.write().remove(&key); + return Ok(true); + } + Err(e) => return Err(std::io::Error::other(e.to_string())), + }; + if crate::client::compute_address(&buf) == key { + // Repaired between the failing read and now. Leave it alone. + return Ok(false); + } + std::fs::remove_file(&path)?; + // The same flush the ordinary delete does, for the same reason: an + // unlink that has not reached the directory can be undone by a power + // loss, and here the entry that comes back is one this node has proven + // wrong. The startup scan would re-index it by name, and the + // known-wrong mark that would otherwise hold it back lives only in + // memory and does not survive the restart, so the node would go back to + // claiming and committing to a chunk it already knows is bad. + if let Some(shard) = path.parent() { + fsync_dir_best_effort(shard); + } + index.write().remove(&key); + Ok(true) + }) + .await; + match outcome { + Ok(Ok(true)) => { + self.clear_known_wrong(address); + self.clear_suspect(address); warn!( - "Chunk {} is in the legacy environment, is not in the file store, and \ - was in neither view; re-queued for copying and the legacy environment \ - stays", - hex::encode(key) + "Removed corrupt chunk file {}; replication will repair it", + hex::encode(address) ); - legacy.only.write().insert(key); - report.unrepairable = report.unrepairable.saturating_add(1); - continue; } - since_log += 1; - if since_log >= VERIFY_LOG_EVERY { - since_log = 0; - info!( - "Pre-retirement verification: {} of at most {total} chunk(s) checked", - report.checked + Ok(Ok(false)) => { + // The re-read hashed and matched: a repair landed between the failing + // read and this one. + self.clear_known_wrong(address); + self.clear_suspect(address); + debug!( + "Chunk {} verified on re-read; leaving it in place", + hex::encode(address) ); } - let outcome = self.verify_one(&legacy, &key).await; - report.checked += 1; - report.bytes += outcome.bytes; - match outcome.verdict { - VerifyVerdict::Intact => {} - VerifyVerdict::Repaired => report.repaired += 1, - VerifyVerdict::Vanished => { - // The file went away while this pass was running, so the key is no - // longer file-backed. Put it back on the copier's list rather than - // republishing it here, where it could resurrect something the - // pruner deleted a moment ago. - legacy.only.write().insert(key); - report.unrepairable += 1; - } - VerifyVerdict::Unrepairable => report.unrepairable += 1, + // Still indexed, so it must not still be claimed: the read that brought us + // here proved the bytes wrong, and the node would otherwise go on committing + // to a chunk it knows it cannot serve. + Ok(Err(e)) => { + self.mark_suspect(address); + warn!( + "Corrupt chunk {} could not be removed: {e}. It stays on disk, and \ + this node stops answering for it.", + hex::encode(address) + ); } - if let Some(delay) = throttle_delay(outcome.bytes, throttle_mib_per_sec) { - tokio::time::sleep(delay).await; + Err(e) => { + self.mark_suspect(address); + warn!("Corrupt-chunk removal task failed: {e}"); } } + } +} - if report.unrepairable == 0 { - info!( - "Pre-retirement verification passed: {} chunk(s) checked, {} repaired", - report.checked, report.repaired - ); - } - // The count this pass started from, and a refusal if the store moved while it - // ran. Retirement compares the same value again immediately before deleting - // anything, so one number covers both windows: during the pass, and after it. - report.health = health_at_start; - if self.files.health_generation() != health_at_start { - warn!( - "A chunk stopped being servable while the pre-retirement pass was running, \ - so this pass does not describe the store. Another runs on the next tick." - ); - report.unrepairable = report.unrepairable.saturating_add(1); - } - Ok(report) - } - - /// Check one chunk that both stores hold, repairing the file if it is wrong. - async fn verify_one(&self, legacy: &Legacy, key: &XorName) -> VerifyOutcome { - // The throttle sleep is deliberately outside this critical section: at 32 MiB/s a - // 4 MiB chunk sleeps for over a tenth of a second, and holding a shard lane for - // that would stall every write to a sixteenth of the address space for hours. - let _lane = self.key_lock(key).await; - - let bytes = match self.files.get_raw(key).await { - Ok(bytes) => bytes, - // Not the same as gone. `Vanished` puts the key back on the copier's list, - // and doing that for a file that is still there and still indexed leaves the - // key in both views at once: the file index keeps it in every commitment, so - // it stays answerable, and an answerable legacy-only key vetoes retirement for - // as long as the process lives. Refuse this pass instead. - Err(e) => { - warn!( - "Chunk {} could not be read while verifying: {e}. The legacy \ - environment stays.", - hex::encode(key) - ); - return VerifyOutcome { - bytes: 0, - verdict: VerifyVerdict::Unrepairable, - }; - } - }; - let len = bytes.as_ref().map_or(0, Vec::len) as u64; - let Some(bytes) = bytes else { - return VerifyOutcome { - bytes: 0, - verdict: VerifyVerdict::Vanished, - }; - }; - if crate::client::compute_address(&bytes) == *key { - // The pass hashed these bytes and they are right, so whatever this store - // thought was wrong with them is not. It reads raw, which does not settle - // that on its own, and leaving the mark would retire the environment while a - // healthy chunk stayed unadvertised until some later verified read. - self.files.note_bytes_proven_good(key); - return VerifyOutcome { - bytes: len, - verdict: VerifyVerdict::Intact, - }; - } - - warn!( - "Chunk {} is in the file store but does not match its address; rewriting it \ - from the legacy environment before that environment is removed", - hex::encode(key) - ); - // Replace in place. Deleting first and writing after would leave a window whose - // only surviving copy is the one this whole pass exists to make safe to delete. - let verdict = match legacy.lmdb.get_raw(key).await { - Ok(Some(good)) if self.files.repair(key, &good).await.is_ok() => { - VerifyVerdict::Repaired - } - _ => { - warn!( - "Chunk {} could not be rewritten from the legacy environment. \ - Retirement stays blocked so its bytes are not thrown away.", - hex::encode(key) - ); - VerifyVerdict::Unrepairable - } - }; - VerifyOutcome { - bytes: len, - verdict, - } - } - - /// Is this verification still worth acting on? - /// - /// # Errors - /// - /// Returns [`Error::Storage`] naming what has changed since the pass ran. - fn proof_is_usable(&self, proof: &VerifyReport) -> Result<()> { - if !proof.is_clean() { - return Err(Error::Storage(format!( - "Refusing to remove the legacy environment: verification reported {} \ - unrepairable chunk(s) (ran: {})", - proof.unrepairable, proof.ran - ))); - } - if !proof.still_describes(&self.files) { - return Err(Error::Storage( - "Refusing to remove the legacy environment: a chunk stopped being \ - servable since it was verified, so that verification no longer describes \ - the file store. A fresh pass runs on the next tick." - .into(), - )); - } - if self.has_pending_writes() { - return Err(Error::Storage( - "Refusing to remove the legacy environment: a write announced itself and \ - has not reported back, so what the environment holds is not yet settled." - .into(), - )); - } - Ok(()) - } - - /// Close the legacy environment and remove it, returning the bytes freed. - /// - /// This is the only destructive step in the migration and the only one that cannot - /// be undone. It is also the only moment the disk comes back. - /// - /// Takes a [`VerifyReport`] rather than a flag so the verification pass cannot be - /// skipped: there is no way to call this without having produced one. - /// - /// # Errors - /// - /// Returns [`Error::Storage`] if verification did not pass or no longer describes the - /// store, if a write has not reported back, if the handle is still shared (the caller - /// should retry on the next tick), or if the directory cannot be removed. - pub async fn retire_legacy( - &self, - proof: &VerifyReport, - still_answerable: &F, - approved_to_shed: &BTreeSet, - ) -> Result - where - F: Fn(&XorName) -> bool + Send + Sync, - { - self.proof_is_usable(proof)?; - // Rechecked here, not only by the caller. Everything between the caller's check - // and this point is a window: the verification pass alone can run for hours, and - // a write whose file half failed inserts a new legacy-only key in the meantime. - if let Some(reason) = self.retirement_blocker(still_answerable) { - return Err(Error::Storage(format!( - "Refusing to remove the legacy environment: {reason}" - ))); - } - // Exclusive from here until the handle is out. Every read, write and delete holds - // this shared, so taking it means none is in progress and none can start: no - // reader is mid-way between discarding a corrupt file and reaching the copy that - // would replace it, and nothing new can start work in an environment that is about - // to go idle. - // - // Released as soon as the handle has been taken and the directory renamed away, - // which is the point after which nothing can reach the environment anyway. The - // deletion that follows can take a long time on a large store, and holding every - // chunk request on the node behind it would turn retirement into an outage. - // - let retiring = self.retirement.write().await; - // Asked again with the guard held, which is the only moment the answer cannot - // change underneath it. The check above can be overtaken by a read that fails - // between there and here. - if !proof.still_describes(&self.files) { - drop(retiring); - return Err(Error::Storage( - "Refusing to remove the legacy environment: a chunk stopped being \ - servable while retirement was starting. A fresh pass runs on the next \ - tick." - .into(), - )); - } - let Some(legacy) = self.legacy() else { - return Ok(0); - }; - let freed = self.legacy_bytes(); - // Let go of our own clone straight away, so the only strong reference that should - // remain is the one the store itself holds. - drop(legacy); - - for attempt in 0..RETIRE_UNWRAP_ATTEMPTS { - // Drained on every attempt, not once up front: `LmdbStorage`'s blocking - // closures capture a cloned `Env` rather than the `Arc`, so the strong count - // alone would not notice a read that is still mapped. The tracker does, and - // it reopens itself, so a read that started since the last drain needs - // another one. - if let Some(l) = self.legacy() { - l.lmdb.wait_idle().await; - drop(l); - } - // Taking the handle out and proving sole ownership happen in the same - // critical section. Deliberately not two steps: taking it first and putting - // it back on failure would leave a window in which reads see no legacy store - // and report a chunk that lives only there as missing. - let taken = { - let mut guard = self.legacy.write(); - match guard.as_ref() { - // A strong count of one means nobody else holds a handle, so nobody - // can be reading the legacy store *or* mutating its key set. That is - // what makes the final check below atomic with the removal: this is - // the only moment at which the answer cannot change underneath us. - Some(l) if Arc::strong_count(&l.lmdb) == 1 => { - // Asked here, in the same critical section as the checks below - // and immediately before the handle is taken. Asking earlier is - // not enough: a write can announce itself under the shared guard, - // be cancelled so the guard is released, and leave its blocking - // half running past the drain above. Its note is the only thing - // that says so, and dropping the journal with the environment - // would take the evidence with it. - if !l.pending.read().is_empty() { - return Err(Error::Storage( - "Refusing to remove the legacy environment: a write \ - announced itself and has not reported back, so what the \ - environment holds is not yet settled." - .into(), - )); - } - let only = l.only.read(); - // A count of one proves nobody else holds a handle, so nobody can - // be mutating this set. That is what makes the two checks below - // authoritative rather than a snapshot that has already moved. - if let Some(key) = only.iter().find(|k| still_answerable(k)) { - return Err(Error::Storage(format!( - "Refusing to remove the legacy environment: chunk {} became \ - answerable again while retirement was in progress", - hex::encode(key) - ))); - } - // Only the keys the caller cleared may go. A write whose file half - // failed adds a legacy-only key that is in no commitment, so the - // answerability check above cannot see it, and it would otherwise - // be destroyed without ever facing the rank, delivery or - // possession gates. - if let Some(key) = only.iter().find(|k| !approved_to_shed.contains(*k)) { - return Err(Error::Storage(format!( - "Refusing to remove the legacy environment: chunk {} entered \ - the legacy-only set after the gates were cleared and has \ - passed none of them", - hex::encode(key) - ))); - } - drop(only); - guard.take() - } - Some(_) => None, - None => return Ok(0), - } - }; - if let Some(Legacy { - lmdb, - only, - pending, - skipped_rollback_copies, - }) = taken - { - drop(only); - drop(pending); - drop(skipped_rollback_copies); - drop(lmdb); - return self.remove_legacy_dir(freed, retiring).await; - } - if attempt + 1 < RETIRE_UNWRAP_ATTEMPTS { - tokio::time::sleep(RETIRE_UNWRAP_BACKOFF).await; - } - } +// ──────────────────────────────────────────────────────────────────────────── +// Free functions +// ──────────────────────────────────────────────────────────────────────────── - // Nothing was taken and nothing will be this tick, so let the node get on with - // serving rather than leaving this held until the function returns. - drop(retiring); - Err(Error::Storage( - "Legacy environment is still being read; retirement deferred to the next tick".into(), +/// Create the destination shard directory if this store has not seen it yet. +/// +/// A newly created directory entry is only durable once its parent is flushed; without +/// that a crash could take the directory and the chunk inside it together. +fn ensure_shard_dir( + chunks_dir: &Path, + dir: &Path, + shard: usize, + present: &parking_lot::Mutex<[bool; SHARD_COUNT]>, +) -> Result<()> { + if present.lock().get(shard).copied().unwrap_or(false) { + return Ok(()); + } + std::fs::create_dir_all(dir).map_err(|e| { + Error::Storage(format!( + "Failed to create shard directory {}: {e}", + dir.display() + )) + })?; + // Load-bearing, like the flush that publishes a chunk into this directory. Until the + // parent is flushed the shard's own entry can be lost, and losing it loses every chunk + // inside it. Reporting the shard present anyway would let the very first chunk written + // into it count as durably stored. + fsync_dir(chunks_dir).map_err(|e| { + Error::Storage(format!( + "Created shard directory {} but could not flush {}: {e}. Not marking the shard \ + usable, because a directory that is not durable cannot hold a chunk that is.", + dir.display(), + chunks_dir.display() )) + })?; + if let Some(slot) = present.lock().get_mut(shard) { + *slot = true; } + Ok(()) +} - /// Remove the legacy directory and record that the migration is over. - /// - /// The handle is already closed by the time this runs, so the node is file-only - /// either way. If the removal fails the phase still moves on, because there is no - /// going back to a half-removed environment, and the operator is told exactly which - /// directory to delete by hand to get the space back. - async fn remove_legacy_dir( - &self, - freed: u64, - retiring: tokio::sync::RwLockWriteGuard<'_, ()>, - ) -> Result { - // Renamed aside first, because `remove_dir_all` is not atomic: a failure partway - // through leaves a directory that can no longer be opened as an environment, and - // recording the migration as finished on top of that would have the node claim - // completion over a half-deleted store. A rename either happens or does not. - let tombstone = free_tombstone_path(&self.config.root_dir); - - if let Err(e) = std::fs::rename(&self.legacy_env_dir, &tombstone) { - // Nothing was deleted, but the handle is already closed, so this node has - // stopped being able to serve anything that lives only in there. Put it back - // rather than carrying on with chunks it holds and cannot read, and rather - // than letting the next tick see no handle and call that success. - let restored = self.reopen_legacy().await; - return Err(Error::Storage(format!( - "Could not move the legacy environment {} aside: {e}. Nothing was deleted{}", - self.legacy_env_dir.display(), - if restored { - " and it has been reopened, so the node keeps serving from both stores." - } else { - ". IT COULD NOT BE REOPENED: this node cannot serve chunks that live only there until it is restarted." - } - ))); - } - // The rename has to reach the directory itself, not just the page cache, and this - // one is not best effort. The tombstone is deleted a few lines below. If the - // rename has not reached the disk when that happens, a power loss brings the - // environment back under its old name with its contents already removed, and the - // next start finds a corrupt environment it cannot open. Stopping here instead - // leaves the tombstone in place, which the next start sweeps. - // Marked from the inside, now that the rename has succeeded and before anything - // is deleted. This is what a directory that reverts to its old name carries with - // it, and it is the only thing a later start treats as permission to delete. - if let Err(e) = mark_directory_retired(&tombstone) { - // Nothing has been deleted and the directory is intact, so put it back rather - // than recording the migration as finished over a store that is still there. - // Recording finished would be worse than it sounds: the next tick restores the - // unmarked directory to its own name, and a node that has already called - // itself file-only would then exit with a live environment on disk and no - // handle to it. - // Only when the mark is provably gone. A mark left inside would have the - // next cleanup pass reap a live, open environment. - let restored = - e.mark_definitely_gone && std::fs::rename(&tombstone, &self.legacy_env_dir).is_ok(); - let reopened = restored && self.reopen_legacy().await; - return Err(Error::Storage(format!( - "Moved the legacy environment to {} but could not mark it retired: {e}. \ - Nothing was deleted{}", - tombstone.display(), - if reopened { - ", and it has been put back, so the node keeps serving from both \ - stores and retirement is tried again." - } else if e.mark_definitely_gone { - ". IT COULD NOT BE PUT BACK: this node cannot serve chunks that live \ - only there until it is restarted." - } else { - ". It has been left where nothing will open it, because a partial \ - retirement mark may still be inside it. Its chunks are in the file \ - store; move it back by hand only after removing that mark." - } - ))); - } - - // Test-only: renamed aside and marked, nothing deleted yet. A process killed here - // is what the recovery on the next start exists for. - #[cfg(any(test, feature = "test-utils"))] - crate::storage::file_store::halt_here_if_asked( - crate::storage::file_store::HALT_AFTER_RETIRE_MARK, - &tombstone, - ); - - if let Err(e) = crate::storage::file_store::fsync_path(&self.config.root_dir) { - warn!( - "The legacy environment was moved aside but {} could not be flushed: {e}. \ - Leaving {} in place rather than deleting a directory whose new name may \ - not have reached the disk. The next start finishes this.", - self.config.root_dir.display(), - tombstone.display() - ); - self.finish_migration(); - return Ok(0); - } - self.finish_migration(); +/// Shard directory index for an address: its last byte. +fn shard_index(address: &XorName) -> usize { + address.last().copied().unwrap_or(0) as usize +} - // From here nothing can reach the environment: its handle is gone and its - // directory is under a name no code looks for. Let the node serve again rather - // than holding every chunk request behind a deletion that can run for minutes. - drop(retiring); +/// Shard directory name for an address: the last two characters of its hex form. +fn shard_name(address: &XorName) -> String { + format!("{:02x}", shard_index(address)) +} - // Only now, and best effort: the bytes come back when this completes, and if it - // does not the next start sweeps the tombstone. - // - // On a detached OS thread, and not awaited. This is a synchronous recursive delete - // of a directory that can hold hundreds of gigabytes and cannot be interrupted - // once it starts. Inside the migration task it would sit through shutdown's grace - // and past it, because an abort is not observed until the call returns; on the - // runtime's blocking pool a normal runtime shutdown would wait for it anyway. A - // plain thread is the only one the process can genuinely walk away from, and the - // directory carries its own retirement mark, so whatever is left is finished by - // the next start. - delete_retired_directory(tombstone); - Ok(freed) - } - - /// Try again to open a legacy environment this node has lost its handle to. - /// - /// A rename that failed and then could not be reopened leaves the directory on disk - /// with no way to read it, and every chunk that lives only there unserved. Saying so - /// once and waiting for a restart is not enough: the reason is usually transient, and - /// a node that is otherwise healthy should not stay half-blind until somebody notices. - /// - /// Returns whether it came back. Does nothing when there is a handle already, or when - /// there is nothing on disk to open. - pub async fn recover_lost_legacy_handle(&self) -> bool { - if self.has_legacy() || !retirement_mark(&self.legacy_env_dir).permits_opening() { - return false; - } - if !legacy_present(&self.config.root_dir).unwrap_or(false) { - return false; - } - // Opened WITHOUT the exclusive guard. Opening scans every key in the environment, - // which on a large store is minutes, and every read and write on the node would - // wait behind it. Nothing else can be installing a handle: retirement does nothing - // while there is none, and this runs from the one migration task. - let (lmdb, legacy_keys) = match Self::open_legacy_env(&self.config).await { - Ok(opened) => opened, - Err(e) => { - warn!( - "Could not reopen {}: {e}. The chunks that live only there stay \ - unreadable until this succeeds.", - self.legacy_env_dir.display() - ); - return false; - } - }; - // Exclusive only to install it, which is instant. - let _recovering = self.retirement.write().await; - if self.has_legacy() { - return false; - } - // The diff happens HERE, not when the environment was read. Reading it takes - // minutes on a large store, and a verifying read in that time can find a file - // rotted and throw it away. With no handle installed there was nothing to put the - // key back into, so a set computed beforehand would be missing it, every gate - // would skip it, and retirement would destroy the intact copy in the environment. - // Under this guard no read, write or delete is in flight, so the file store's - // answer cannot move while it is being asked. - let only = Self::keys_only_in_legacy(&legacy_keys, &self.files); - *self.legacy.write() = Some(Legacy { - lmdb, - only: Arc::new(parking_lot::RwLock::new(only)), - skipped_rollback_copies: Arc::new(std::sync::atomic::AtomicU64::new(0)), - pending: Arc::new(parking_lot::RwLock::new(BTreeMap::new())), - }); - // A node that recorded itself file-only and then got an environment back has to - // go through the migration again from the start: the phase decides what the - // driver does, and file-only does no copying, so leaving it there would give the - // node a handle it never uses. Conservative on purpose; the copier finds most of - // the work already done. - if self.migration_phase() == MigrationPhase::FilesOnly { - warn!( - "Recovered a legacy chunk environment after recording this node as \ - file-only. Starting the migration again from the copying stage." - ); - let mut state = self.state.write(); - state.phase = MigrationPhase::Bridging; - state.committed_at_unix = None; - state.rebuilds_since_commit = 0; - let snapshot = state.clone(); - drop(state); - if let Err(e) = snapshot.save(&self.config.root_dir) { - warn!("Could not persist the migration marker: {e}"); - } - } - warn!( - "Reopened {} after losing its handle", - self.legacy_env_dir.display() - ); - true - } +/// True for a string of hex digits in either case. +fn is_hex_any_case(s: &str) -> bool { + !s.is_empty() && s.bytes().all(|b| b.is_ascii_hexdigit()) +} - /// Is there a retired directory still waiting to be deleted? - /// - /// Separate from having a legacy environment: a node whose removal was interrupted has - /// no handle and nothing to migrate, but its disk has not come back. Something has to - /// keep trying during this uptime rather than leaving it until the next restart. - #[must_use] - pub fn has_cleanup_pending(&self) -> bool { - !retired_tombstones(&self.config.root_dir).is_empty() - || !retirement_mark(&self.legacy_env_dir).permits_opening() +/// Move an entry aside under a name that can never be read as a chunk. +fn quarantine_entry(path: &Path) { + let aside = path.with_extension("not-a-chunk"); + match std::fs::rename(path, &aside) { + Ok(()) => warn!( + "Chunk store: moved {} aside to {}; a name that differs from a chunk name only \ + by case collides with it on Windows and macOS", + path.display(), + aside.display() + ), + Err(e) => warn!( + "Chunk store: {} collides with a chunk name by case folding and could not be \ + moved aside: {e}. Rename or delete it.", + path.display() + ), } +} - /// Try again to finish a removal a previous attempt left behind. - /// - /// Safe to call at any time: it only ever moves or deletes a directory that carries - /// its own retirement mark. - pub fn retry_cleanup(&self) { - // Never while this node has the environment open. Cleanup decides what to do from - // the directory's own mark, and a mark that outlived a failed retirement would - // have it rename a live, mapped environment out from under the handle. - if self.has_legacy() { - sweep_retired_legacy(&self.config.root_dir); - return; - } - finish_interrupted_retirement(&self.config.root_dir); - } +/// True for a string of lowercase hex digits only. +fn is_lower_hex(s: &str) -> bool { + !s.is_empty() && s.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')) +} - /// Resolve writes whose outcome was never recorded. - /// - /// A write announces itself before it starts and clears the note when both halves - /// have returned. A note still there afterwards belongs to a write nobody waited for, - /// and only the disk can say what became of it: the file store has the chunk, or the - /// environment does and nothing else, or neither and there was never anything to - /// protect. - pub async fn reconcile_pending_writes(&self) { - if !self.has_pending_writes() { - return; - } - // Exclusively, and before the snapshot. Draining is not a barrier on its own: - // writes hold this shared, and a new one for the same key could announce itself, - // be cancelled, and leave its blocking half running while this decided the older - // one's fate and removed the single entry they share. Held here, nothing new can - // start, so what the disk says once the drain returns is final. - // - // Only reached when something is waiting, which after a clean run is never, so - // this is not a stall on the ordinary path. - let _settling = self.retirement.write().await; - let Some(legacy) = self.legacy() else { - return; - }; - let waiting: Vec = legacy.pending.read().keys().copied().collect(); - if waiting.is_empty() { - return; - } - legacy.lmdb.wait_idle().await; - self.files.wait_idle().await; - for key in waiting { - let _lane = self.key_lock(&key).await; - if self.files.is_indexed(&key) { - legacy.only.write().remove(&key); - legacy.pending.write().remove(&key); - continue; - } - match legacy.lmdb.get_raw(&key).await { - Ok(Some(_)) => { - debug!( - "Chunk {} was written to the legacy environment by a call that \ - never returned; recording it so the copier picks it up", - hex::encode(key) - ); - legacy.only.write().insert(key); - legacy.pending.write().remove(&key); - } - // Nothing behind it: there was never anything to protect. - Ok(None) => { - legacy.pending.write().remove(&key); - } - // NOT the same as nothing behind it. Dropping the note on a read that - // failed would leave a committed write with no protection at all, which - // is the case this journal exists for. Keep it and ask again next tick; - // retirement stays vetoed meanwhile. - Err(e) => warn!( - "Could not tell what became of the write for {}: {e}. Asking again on \ - the next tick.", - hex::encode(key) - ), - } - } +/// Decode a filename back into the address it names, or `None` if it is not one. +/// +/// Rejects uppercase deliberately. On a case-folding filesystem (NTFS, default APFS) +/// accepting both cases would let one file answer to two index entries. +fn decode_chunk_name(name: &str) -> Option { + if name.len() != CHUNK_NAME_LEN || !is_lower_hex(name) { + return None; } + let bytes = hex::decode(name).ok()?; + XorName::try_from(bytes.as_slice()).ok() +} - /// Are there writes in flight whose outcome nothing has recorded? - #[must_use] - pub fn has_pending_writes(&self) -> bool { - self.legacy().is_some_and(|l| !l.pending.read().is_empty()) +/// Flush a directory so a rename or creation inside it survives power loss. +/// +/// Best effort by design. Linux and XFS require it, macOS accepts it with undocumented +/// effect, and Windows offers no way to do it at all through the standard library. The +/// content is content-addressed and re-replicable, so a lost directory entry costs a +/// refetch rather than data. Pretending otherwise in the code would be dishonest. +#[cfg(unix)] +fn fsync_dir_best_effort(path: &Path) { + if let Err(e) = fsync_dir(path) { + debug!("Directory flush of {} failed: {e}", path.display()); } +} - /// Is there anything at the legacy environment's path at all? - /// - /// Asked without a handle, and answered conservatively: a path this node cannot even - /// look at counts as present. The migration is not finished while something is there, - /// whether or not this node can currently read it. - #[must_use] - pub fn legacy_dir_is_on_disk(&self) -> bool { - // `symlink_metadata`, not `try_exists`, which follows links. An operator's link to - // storage that is not mounted right now reads as nothing at all through the - // second, and the node would call its migration finished and go file-only, blind - // to every chunk that lives only there until somebody restarts it. - match std::fs::symlink_metadata(&self.legacy_env_dir) { - Ok(_) => true, - // Only "it is not there" means it is not there. A permission change or a - // transient fault is an unanswered question, and answering it with "nothing - // here" is how the driver declares the migration finished over a store it has - // merely lost sight of. - Err(e) => e.kind() != std::io::ErrorKind::NotFound, - } - } +/// Flush a directory, reporting whether it worked. +/// +/// Used where the answer is load-bearing: a chunk copied out of the legacy store is only +/// durable once its directory entry is, and that copy is what permits the legacy store to +/// be deleted. +#[cfg(unix)] +fn fsync_dir(path: &Path) -> std::io::Result<()> { + File::open(path)?.sync_all() +} - /// Is the legacy environment a link this node must not delete? - /// - /// Copying out of it works; only the removal is refused. Callers use this to stop - /// waiting for a retirement that is never going to happen. - #[must_use] - pub fn legacy_is_a_link(&self) -> bool { - self.has_legacy() && is_a_link(&self.legacy_env_dir) - } +/// Off Unix there is no way to flush a directory through the standard library, so this +/// reports success without being able to promise anything. +/// +/// That is why the publish path off Unix does not use a rename at all: it creates the +/// chunk under its final name and flushes the file, which Microsoft documents as flushing +/// the creation metadata with it. Directory creation has no equivalent, so the guarantee +/// there rests on the pre-retirement pass, which re-reads every chunk before the legacy +/// store is deleted, and on the operator gate that keeps retirement off a platform until +/// forced power loss has been shown to hold old-or-new on it. +/// +/// Returns a `Result` so the callers that must handle a flush failure on Unix read the +/// same on every platform. +#[cfg(not(unix))] +#[allow(clippy::unnecessary_wraps)] +fn fsync_dir(_path: &Path) -> std::io::Result<()> { + Ok(()) +} - /// How many writes this node made without a rollback copy, cumulatively. - /// - /// Zero on a node that is not bridging, and zero on a bridging node whose environment - /// has room. A number that is climbing says this node would lose those chunks on a - /// rollback to a pre-migration build, which is a fleet question the second release - /// turns on and which a per-chunk log line cannot answer. - /// - /// Attempts rather than distinct chunks, for the reason given on the field: it is a - /// rate, not an inventory. - #[must_use] - pub fn writes_without_a_rollback_copy(&self) -> u64 { - self.legacy().map_or(0, |l| { - l.skipped_rollback_copies - .load(std::sync::atomic::Ordering::Relaxed) - }) +/// No-op on platforms with no way to flush a directory handle. +#[cfg(not(unix))] +fn fsync_dir_best_effort(_path: &Path) {} + +/// Warn if the deepest chunk path this store can produce is close to `MAX_PATH`. +#[cfg(windows)] +fn check_path_budget(chunks_dir: &Path) { + // Measured absolute, because that is what the filesystem sees. A relative root is the + // case that still fails hard at MAX_PATH, since the standard library's long-path + // handling only applies to paths it resolves as absolute. + let absolute = if chunks_dir.is_absolute() { + chunks_dir.to_path_buf() + } else { + std::env::current_dir() + .map_or_else(|_| chunks_dir.to_path_buf(), |cwd| cwd.join(chunks_dir)) + }; + // `{chunks_dir}\{xy}\{64 hex}` — two separators, two shard characters, 64 name + // characters. + let deepest = absolute.as_os_str().len() + 1 + 2 + 1 + CHUNK_NAME_LEN; + if deepest > WINDOWS_PATH_WARN_LEN { + warn!( + "Chunk file paths will be {deepest} characters, close to the {} character \ + Windows limit. Move the node root closer to the drive letter if writes start \ + failing.", + WINDOWS_PATH_WARN_LEN + ); } +} - /// Is there an environment on disk this node cannot classify at all? - /// - /// Neither removable nor openable, which is not a state waiting will clear: something - /// about the path has to change first, and until it does the node will refuse to touch - /// it in either direction. The driver treats this the way it treats a lost handle or a - /// link, by standing down from the shared volume and saying so where an operator looks, - /// because holding a disk exclusively to wait for a person is a disk nobody else can - /// use. - #[must_use] - pub fn legacy_cannot_be_classified(&self) -> bool { - retirement_mark(&self.legacy_env_dir) == RetirementMark::Unknown - } +/// No-op where path length is not a practical constraint. +#[cfg(not(windows))] +fn check_path_budget(_chunks_dir: &Path) {} - /// Is there an environment on disk this node can no longer read? - #[must_use] - pub fn has_lost_its_legacy_handle(&self) -> bool { - !self.has_legacy() - && retirement_mark(&self.legacy_env_dir).permits_opening() - // Conservative in the same direction as the retirement blocker, which reads the - // same failure as "there is one". A question that cannot be answered is not an - // answer of no, and answering no here left the node holding the shared volume - // for the six-hour cap over work no amount of disk will finish. - && legacy_present(&self.config.root_dir).unwrap_or(true) - } +/// Write `bytes` to `path` so a reader sees either the old content or the new. +/// Is this the exact name [`write_file_atomic`] gives its temporaries? +/// +/// `.tmp..<8 hex>.marker`, with both middle parts checked. Matching on the prefix and +/// suffix alone would also take `.tmp.operator-notes.marker`, and this runs over a +/// directory holding a node's data, so what it removes is not a place to be approximate. +fn is_marker_temp_name(name: &str) -> bool { + let Some(rest) = name.strip_prefix(TEMP_PREFIX) else { + return false; + }; + let Some(rest) = rest.strip_suffix(".marker") else { + return false; + }; + let mut parts = rest.split('.'); + let (Some(pid), Some(nonce), None) = (parts.next(), parts.next(), parts.next()) else { + return false; + }; + !pid.is_empty() + && pid.bytes().all(|b| b.is_ascii_digit()) + && nonce.len() == 8 + && nonce.bytes().all(|b| b.is_ascii_hexdigit()) +} - /// Reopen the legacy store after a failed retirement, so the node keeps serving. - /// - /// Returns whether it came back. The handle is closed before the rename is attempted, - /// so a rename that fails leaves the node holding chunks it can no longer read; that - /// is worth undoing rather than living with until the next restart. - async fn reopen_legacy(&self) -> bool { - if !legacy_present(&self.config.root_dir).unwrap_or(false) { - return false; - } - match Self::open_legacy(&self.config, &self.files).await { - Ok(legacy) => { - *self.legacy.write() = Some(legacy); - warn!("Reopened the legacy chunk environment after a failed retirement"); - true - } - Err(e) => { - error!("Could not reopen the legacy chunk environment: {e}"); - false - } +/// Remove marker temporaries a previous run left beside `path`. +/// +/// `write_file_atomic` writes its temporary next to its target. For the layout marker +/// that is inside `chunks/`, which the startup scan sweeps; for the migration marker it is +/// the node root, which nothing sweeps, so a crash between the write and the rename leaves +/// one there for the life of the node. Each is a few hundred bytes, so this is inodes +/// rather than capacity, but nothing else was ever going to remove them. +/// +/// Only the exact shape this module writes, and only files: a name has to carry the temp +/// prefix and the marker suffix. Anything broader would be this function deciding what +/// else in a node's root directory is rubbish, which is not its business. +/// +/// Best effort throughout. Failing to tidy up is not a reason to refuse to start, and the +/// caller takes the store lock before this runs, so there is no other process whose live +/// temporary this could take. +pub fn sweep_marker_temps(dir: &Path) { + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + for entry in entries.flatten() { + let name = entry.file_name(); + let Some(name) = name.to_str() else { + continue; + }; + if !is_marker_temp_name(name) { + continue; } - } - - /// Record that this node serves from files alone from here on. - fn finish_migration(&self) { - self.files.invalidate_capacity_cache(); - { - let mut state = self.state.write(); - state.phase = MigrationPhase::FilesOnly; + if !entry.file_type().is_ok_and(|kind| kind.is_file()) { + continue; } - let snapshot = self.state.read().clone(); - if let Err(e) = snapshot.save(&self.config.root_dir) { - warn!("Could not persist the migration marker: {e}"); + match std::fs::remove_file(entry.path()) { + Ok(()) => debug!( + "Swept a leftover marker temporary {}", + entry.path().display() + ), + Err(e) => debug!("Could not sweep {}: {e}", entry.path().display()), } } } -/// What checking one chunk concluded. -enum VerifyVerdict { - /// The file matches its name. - Intact, - /// The file was wrong and was rewritten from the legacy copy. - Repaired, - /// The file was wrong and could not be rewritten. - Unrepairable, - /// The file disappeared while the pass was running. - Vanished, +fn write_file_atomic(path: &Path, bytes: &[u8]) -> Result<()> { + let Some(dir) = path.parent() else { + return Err(Error::Storage(format!( + "Refusing to write {} — it has no parent directory", + path.display() + ))); + }; + let temp = dir.join(format!( + "{TEMP_PREFIX}{}.{:08x}.marker", + std::process::id(), + rand::random::() + )); + write_temp(&temp, bytes)?; + // Through the retry, because these small files (the layout marker, the migration + // state) are rewritten while the node runs, and on Windows a scanner holding a handle + // for a few milliseconds turns an ordinary rewrite into a hard failure. + rename_with_retry(&temp, path).map_err(|e| { + let _ = std::fs::remove_file(&temp); + Error::Storage(format!("Failed to publish {}: {e}", path.display())) + })?; + fsync_dir_best_effort(dir); + Ok(()) } -/// One chunk's verification result. -struct VerifyOutcome { - /// Bytes read, for the throttle. - bytes: u64, - /// What was concluded. - verdict: VerifyVerdict, +/// Read the layout marker, writing the current one if the store is new. +fn read_or_write_layout(chunks_dir: &Path) -> Result { + let path = chunks_dir.join(LAYOUT_FILE_NAME); + match read_small_file(&path) { + Ok(bytes) => serde_json::from_slice(&bytes).map_err(|e| { + Error::Storage(format!( + "Chunk store layout marker {} is unreadable: {e}. Refusing to open rather \ + than guess the layout.", + path.display() + )) + }), + Err(e) if e.kind() == ErrorKind::NotFound => { + if store_has_entries(chunks_dir) { + warn!( + "Chunk store at {} has data but no layout marker. Adopting it under \ + the current scheme, which is the only one this build implements. If \ + it was written by a build with a different layout its chunks will \ + appear to be missing.", + chunks_dir.display() + ); + } + let layout = StoreLayout::default(); + let bytes = serde_json::to_vec_pretty(&layout) + .map_err(|e| Error::Storage(format!("Failed to encode chunk store layout: {e}")))?; + write_file_atomic(&path, &bytes)?; + debug!("Wrote chunk store layout marker to {}", path.display()); + Ok(layout) + } + Err(e) => Err(Error::Storage(format!( + "Failed to read chunk store layout marker {}: {e}", + path.display() + ))), + } } -/// What the pre-retirement verification pass found. -/// -/// Every field is private, and the only way to obtain one is -/// [`ChunkStore::verify_before_retire`]. That is deliberate: it is the sole evidence -/// [`ChunkStore::retire_legacy`] accepts that the file store really holds what it claims, -/// and a report anyone could construct would be no evidence at all. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -pub struct VerifyReport { - /// Whether the pass actually ran. - ran: bool, - /// Chunks re-hashed. - checked: u64, - /// Bytes read. - bytes: u64, - /// Chunks whose file was wrong and was rewritten from the legacy copy. - repaired: u64, - /// Chunks whose file was wrong and could not be repaired. - unrepairable: u64, - /// What the file store's health looked like when this pass finished. - /// - /// A clean report is reused for a while rather than re-read on every tick, and a lot - /// can happen in that window: a kept file can start failing to read while ordinary - /// requests are served from the legacy copy, and the node would then delete the - /// legacy copy on the strength of a pass that no longer describes the store. This is - /// how retirement tells, immediately before it deletes anything. - health: u64, +/// Whether the store directory already holds at least one shard. +fn store_has_entries(chunks_dir: &Path) -> bool { + let Ok(entries) = std::fs::read_dir(chunks_dir) else { + return false; + }; + entries.filter_map(std::result::Result::ok).any(|e| { + e.file_name() + .to_str() + .is_some_and(|n| n.len() == 2 && is_lower_hex(n)) + }) } -impl VerifyReport { - /// Whether this report clears the way for retirement. - #[must_use] - pub fn is_clean(&self) -> bool { - self.ran && self.unrepairable == 0 - } - - /// Does this report still describe the store? - #[must_use] - fn still_describes(&self, files: &FileStore) -> bool { - self.health == files.health_generation() - } - - /// Chunks re-hashed. - #[must_use] - pub fn checked(&self) -> u64 { - self.checked - } - - /// Chunks rewritten from the legacy copy. - #[must_use] - pub fn repaired(&self) -> u64 { - self.repaired - } - - /// Chunks that could not be made good. - #[must_use] - pub fn unrepairable(&self) -> u64 { - self.unrepairable - } -} +/// Largest a metadata marker may be before it is treated as corrupt. +const MAX_MARKER_BYTES: u64 = 64 * 1024; -/// Mark a retired environment directory as retired, from the inside, durably. +/// Read a small metadata file, refusing an implausibly large one. /// -/// Called only after the directory has already been renamed aside, so it can never land -/// inside a live environment. See [`RETIRED_MARKER`] for why it goes inside. +/// The chunk path is bounded for exactly this reason; the markers live in the same data +/// directory and deserve the same ceiling. /// /// # Errors /// -/// Returns [`Error::Storage`] if it cannot be created or flushed. -fn mark_directory_retired(dir: &Path) -> std::result::Result<(), MarkFailure> { - match write_retirement_mark(dir) { - Ok(()) => Ok(()), - Err(e) if e.pre_existing => { - // Nothing here was created by this attempt, so there is nothing to take back. - // Removing a mark that was already there because re-flushing it failed is how - // a correctly retired directory comes to look unmarked, and an unmarked - // directory is restored as a live environment. - Err(MarkFailure { - pre_existing: true, - ..e - }) - } - Err(e) => { - // A half-written mark is worse than none: the caller puts the directory back - // under the live name and reopens it, and a mark left inside would have the - // next cleanup pass reap a live, open environment. If it cannot be taken away, - // say so, and the caller keeps the directory where nothing will open it. - let path = dir.join(RETIRED_MARKER); - match std::fs::remove_file(&path) { - Ok(()) => {} - Err(gone) if gone.kind() == std::io::ErrorKind::NotFound => {} - Err(stuck) => { - return Err(MarkFailure { - reason: format!( - "{e}. The partial mark at {} could not be removed either \ - ({stuck})", - path.display() - ), - mark_definitely_gone: false, - pre_existing: false, - }) - } - } - if let Err(flush) = crate::storage::file_store::fsync_path(dir) { - return Err(MarkFailure { - reason: format!( - "{e}. Removing the partial mark at {} could not be flushed \ - ({flush})", - path.display() - ), - mark_definitely_gone: false, - pre_existing: false, - }); - } - Err(MarkFailure { - reason: format!("{e}"), - mark_definitely_gone: true, - pre_existing: false, - }) - } - } -} - -/// Why a directory could not be marked retired, and whether it is safe to reopen. -#[derive(Debug)] -struct MarkFailure { - /// What went wrong, for the operator. - reason: String, - /// Is the directory provably free of a partial mark? - /// - /// Only then may the caller put it back under the live name. A mark left inside would - /// have the next cleanup pass reap a live, open environment. - mark_definitely_gone: bool, - /// Was the mark already there before this attempt? - /// - /// Then this attempt created nothing and must take nothing away. Removing a mark that - /// was already there because re-flushing it failed is how a correctly retired - /// directory comes to look unmarked, and an unmarked directory is restored as a live - /// environment. - pre_existing: bool, -} - -impl std::fmt::Display for MarkFailure { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.reason) +/// Returns an I/O error, including `NotFound`, so callers can distinguish "no marker yet". +pub fn read_small_file(path: &Path) -> std::io::Result> { + let file = File::open(path)?; + let mut bytes = Vec::new(); + let read = file.take(MAX_MARKER_BYTES + 1).read_to_end(&mut bytes)?; + if read as u64 > MAX_MARKER_BYTES { + return Err(std::io::Error::other(format!( + "{} is larger than the {MAX_MARKER_BYTES} byte limit for a marker file", + path.display() + ))); } + Ok(bytes) } -/// Create the mark. See [`mark_directory_retired`], which owns the failure handling. -fn write_retirement_mark(dir: &Path) -> std::result::Result<(), MarkFailure> { - let path = dir.join(RETIRED_MARKER); - let mut file = match std::fs::OpenOptions::new() +/// Take the store lock, or refuse to open the store. +/// +/// Both failures are refusals, deliberately. Unlike LMDB, which was genuinely +/// multi-process safe, two of these stores on one directory keep independent in-memory +/// indices, independent views of what is in flight, and independent opinions about +/// whether the legacy environment may be deleted: both would report the same write as +/// new and each would keep serving keys the other had deleted. A node that cannot create +/// the lock file has no way to know it is alone, and this is the one migration where +/// being wrong about that destroys data. +/// +/// The lock is an [`Arc`] so the work that relies on it can hold a lease. The startup +/// scan sweeps interrupted writes on the strength of being alone in the directory, and it +/// runs on a thread that outlives the future that started it. +/// +/// # Errors +/// +/// Returns [`Error::Storage`] when another process owns the directory, or when the lock +/// file cannot be created. +fn acquire_store_lock(chunks_dir: &Path) -> Result> { + let path = chunks_dir.join(LOCK_FILE_NAME); + let file = match OpenOptions::new() .write(true) - .create_new(true) + .create(true) + .truncate(false) .open(&path) { Ok(f) => f, - // Already there, from an attempt that got this far and no further. Flushed - // again rather than taken on trust: the attempt that wrote it may have been the - // one that could not flush it. - Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => { - // Something is already at that name. That it could not be created is not the - // same as its being a mark this node can read, and everything downstream - // deletes an environment on the strength of it. The rest of this file insists - // the name is not the evidence; this is the one place that was taking it. - if retirement_mark(dir) != RetirementMark::Present { - return Err(MarkFailure { - reason: format!( - "{} already exists but cannot be read as a retirement mark, so it \ - is not one this node will delete on. Check what is at that path.", - path.display() - ), - mark_definitely_gone: false, - pre_existing: true, - }); - } - return crate::storage::file_store::fsync_path(dir).map_err(|flush| MarkFailure { - reason: format!( - "{} is already there but could not be flushed: {flush}", - path.display() - ), - mark_definitely_gone: false, - pre_existing: true, - }); - } + // Not a warning and carry on. Without this lock two processes can open the same + // directory, each with its own index, its own view of what is in flight, and its + // own opinion about whether the legacy environment may be deleted. A node that + // cannot take it has no way to know it is alone, and this is the one migration + // where being wrong about that destroys data. Err(e) => { - return Err(MarkFailure { - reason: format!("Could not mark {} as retired: {e}", path.display()), - mark_definitely_gone: true, - pre_existing: false, - }) + return Err(Error::Storage(format!( + "Could not create the chunk store lock {}: {e}. Refusing to start: \ + without it this node cannot tell whether another is using the same data \ + directory. Fix the permissions on that path, or remove a stale lock file \ + left by a different user.", + path.display() + ))) } }; - // For whoever reads the directory. To the node, presence is the whole signal. - if let Err(e) = file.write_all( - b"This chunk environment was verified as fully copied into the file store and \n\ -retired. It is being deleted; if it is still here, that was interrupted and the next \n\ -node start finishes it. Nothing needs it.\n", - ) { - drop(file); - return Err(MarkFailure { - reason: format!("Could not write {}: {e}", path.display()), - mark_definitely_gone: false, - pre_existing: false, - }); + match file.try_lock_exclusive() { + Ok(()) => Ok(Arc::new(file)), + Err(e) => Err(Error::Storage(format!( + "Another process already has the chunk store at {} open ({e}). Two nodes \ + cannot share one data directory: each keeps its own index and they would \ + disagree about what is stored. Stop the other node first.", + chunks_dir.display() + ))), } - file.sync_all().map_err(|e| MarkFailure { - reason: format!( - "Could not flush {}: {e}. Not deleting on the strength of a mark that may not \ - survive a power loss.", - path.display() - ), - mark_definitely_gone: false, - pre_existing: false, - })?; - // And the directory that now contains it. Flushing the file makes its contents - // durable; the entry naming it is in the directory, and on Unix that needs its own - // flush. Without this the mark can be missing after a crash from a directory that - // was in fact retired, which is the whole question this file answers. - crate::storage::file_store::fsync_path(dir).map_err(|e| MarkFailure { - reason: format!( - "Marked {} retired but could not flush {}: {e}. Not deleting on the strength \ - of a mark that may not survive a power loss.", - path.display(), - dir.display() - ), - mark_definitely_gone: false, - pre_existing: false, - }) } -/// Delete a retired directory in the background, without anything waiting for it. -/// -/// The caller is finished with it either way: the environment is closed, the directory is -/// under a name nothing looks for, and it carries its own mark, so an interrupted deletion -/// is finished by the next start. What matters is that neither shutdown nor startup ever -/// blocks on a recursive delete that can run for minutes. -fn delete_retired_directory(dir: PathBuf) { - // One at a time per directory. The driver asks for cleanup on every tick while - // anything is pending, and starting a fresh thread each time would leave hundreds of - // them asleep on the same path, all retrying the same failure. - if !REAPING.lock().insert(dir.clone()) { - return; - } - let named = dir.clone(); - let started = std::thread::Builder::new() - .name("chunk-store-retire".into()) - .spawn(move || { - let _done = ReapingGuard(dir.clone()); - for attempt in 1..=RETIRED_DELETE_ATTEMPTS { - match remove_marked_directory(&dir) { - Ok(()) => { - info!( - migration_event = "space_returned", - "Removed the retired chunk environment {} and returned its \ - space", - dir.display() - ); - return; - } - // Worth another go: on Windows a scanner or an antivirus can hold a - // handle inside it for a moment, and a partial delete leaves less to - // do next time. - Err(e) if attempt < RETIRED_DELETE_ATTEMPTS => { - debug!( - "Could not delete {} (attempt {attempt}): {e}. Trying again.", - dir.display() - ); - std::thread::sleep( - (RETIRED_DELETE_BACKOFF * attempt).min(RETIRED_DELETE_BACKOFF_MAX), - ); - } - Err(e) => warn!( - "The chunk environment has been retired but {} could not be \ - deleted: {e}. Its space is not returned until it is, and the node \ - needs nothing from it. The next start tries again.", - dir.display() - ), - } - } - }); - if let Err(e) = started { - REAPING.lock().remove(&named); - warn!( - "Could not start the thread to delete the retired chunk environment {}: {e}. \ - The next start sweeps it.", - named.display() - ); - } +/// What a startup scan found. +struct ScanResult { + /// Every published address, ascending. + keys: Vec, + /// Which shard directories already exist. + shards_present: [bool; SHARD_COUNT], + /// Orphaned temp files removed. + swept_temps: usize, + /// Entries that were neither a chunk nor one of ours. + skipped: usize, } -/// Directories a reaper thread is already working on. -static REAPING: parking_lot::Mutex> = parking_lot::Mutex::new(BTreeSet::new()); - -/// Releases a directory from [`REAPING`] however its thread ends. -struct ReapingGuard(PathBuf); +/// Rebuild the key set from directory entries. +/// +/// Reads names only. A `stat` per entry costs about ten times the enumeration on Linux +/// and macOS and fifty to sixty times on Windows, and buys nothing: the filename is the +/// key, and the content is verified on read. +fn scan_store(chunks_dir: &Path) -> Result { + let mut result = ScanResult { + keys: Vec::new(), + shards_present: [false; SHARD_COUNT], + swept_temps: 0, + skipped: 0, + }; -impl Drop for ReapingGuard { - fn drop(&mut self) { - REAPING.lock().remove(&self.0); - } -} + let top = std::fs::read_dir(chunks_dir).map_err(|e| { + Error::Storage(format!( + "Failed to enumerate chunk store {}: {e}", + chunks_dir.display() + )) + })?; -/// Delete a retired directory, taking its mark away last of all. -/// -/// `remove_dir_all` walks in whatever order the filesystem hands back, so it can unlink -/// the mark and then fail on the next entry, which is exactly what a Windows sharing -/// violation on the data file produces. What is left is a genuinely retired, partly -/// deleted directory carrying no evidence that it was retired, and the next start would -/// read that as an intact environment and restore it. -/// -/// Emptying it first and removing the mark last means the mark is only ever absent from a -/// directory that has nothing else left in it. -/// -/// # Errors -/// -/// Returns the underlying I/O error. The directory is left with its mark intact on every -/// failure that happens before the mark is reached. -fn remove_marked_directory(dir: &Path) -> std::io::Result<()> { - // Never through a link. An operator who points the chunk environment at another - // volume leaves a symlink here, and walking it would delete the contents of a - // directory that is not this node's to delete. Retirement refuses such a root before - // it gets this far; this is the second line, because the check and the walk are not - // one operation. - if std::fs::symlink_metadata(dir)?.file_type().is_symlink() { - return Err(std::io::Error::other(format!( - "{} is a link, not a directory. Refusing to delete through it.", - dir.display() - ))); - } - for entry in std::fs::read_dir(dir)? { - let entry = entry?; - if entry.file_name() == RETIRED_MARKER { + for entry in top { + let entry = entry.map_err(|e| { + Error::Storage(format!( + "Failed to read an entry of {}: {e}", + chunks_dir.display() + )) + })?; + let name = entry.file_name(); + let Some(name) = name.to_str() else { + result.skipped = result.skipped.saturating_add(1); + continue; + }; + if name == LAYOUT_FILE_NAME || name == LOCK_FILE_NAME { continue; } - let path = entry.path(); - if entry.file_type()?.is_dir() { - std::fs::remove_dir_all(&path)?; - } else { - std::fs::remove_file(&path)?; - } - } - match std::fs::remove_file(dir.join(RETIRED_MARKER)) { - Ok(()) => {} - Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} - Err(e) => return Err(e), - } - match std::fs::remove_dir(dir) { - Ok(()) => Ok(()), - Err(e) => { - // The mark is gone and the directory is not, which is the one state the whole - // scheme says cannot happen: a start that found it would read an unmarked - // directory as an intact environment. Put the mark back before giving up. - if let Err(remark) = mark_directory_retired(dir) { - error!( - "Could not remove {} ({e}) and could not restore its retirement mark \ - ({remark}). It is empty and nothing needs it; delete it by hand.", - dir.display() - ); + if name.starts_with(TEMP_PREFIX) { + if sweep_temp(&entry.path()) { + result.swept_temps = result.swept_temps.saturating_add(1); } - Err(e) + continue; } - } -} - -/// What a directory's own contents say about whether it was retired. -/// -/// Three answers, not two. Reading the mark can fail for reasons that are neither yes nor -/// no: a permission change, a descriptor limit, a filesystem that has gone away underneath -/// the node. Folding that into "no" is the failure mode the rest of this file exists to -/// avoid, and it fails in the worst direction: a retired environment that reads as unmarked -/// is put back under the live name and reopened, and its keys re-enter a commitment they -/// have already left. -/// -/// So an unreadable answer is its own answer, and the two questions callers actually ask -/// are asked separately. Neither of them treats "cannot tell" as permission. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum RetirementMark { - /// The directory carries its mark. It is the remains of a removal. - Present, - /// The directory carries no mark, and that is known rather than assumed. - Absent, - /// Whether it carries one could not be determined. - Unknown, -} - -impl RetirementMark { - /// May this directory be deleted, or treated as already gone? - /// - /// Only a mark actually read says yes. Deleting on a guess destroys chunks. - const fn permits_removal(self) -> bool { - matches!(self, Self::Present) + if name.len() != 2 || !is_lower_hex(name) { + warn!( + "Chunk store: ignoring unexpected entry {name} in {}", + chunks_dir.display() + ); + result.skipped = result.skipped.saturating_add(1); + continue; + } + let Ok(shard) = u8::from_str_radix(name, 16) else { + result.skipped = result.skipped.saturating_add(1); + continue; + }; + // `shards_present` is set inside `scan_shard`, on success only. Setting it from + // the name alone would make a stray regular file called `ab` look like a shard + // that already exists, and every write to that shard would then fail with a + // misleading error until the node was restarted. + scan_shard(&entry.path(), shard, &mut result)?; } - /// May this directory be opened and served from? - /// - /// Only a mark known to be absent says yes. Opening a retired environment puts keys - /// back into a commitment they have already left. - const fn permits_opening(self) -> bool { - matches!(self, Self::Absent) - } + result.keys.sort_unstable(); + result.keys.dedup(); + Ok(result) } -/// Has this directory been retired? -/// -/// A link is never treated as retired, whatever it points at: the mark would have been -/// written through it into somebody else's directory, and acting on it would delete -/// somebody else's data. A path whose kind cannot be determined is not a link either way, -/// and is reported as unknown rather than as a link, so that neither question gets a yes. -fn retirement_mark(dir: &Path) -> RetirementMark { - match std::fs::symlink_metadata(dir) { - Ok(meta) if meta.file_type().is_symlink() => return RetirementMark::Absent, - Ok(_) => {} - // Nothing here at all, which is the ordinary case on a node that has already - // finished or never had a legacy store. There is no mark because there is nothing - // to carry one, and that is known rather than undetermined. - Err(e) if e.kind() == std::io::ErrorKind::NotFound => return RetirementMark::Absent, - Err(e) => { - // Debug, not warn: this is asked on every tick, so a warn here would be a - // wall of the same line. The operator-facing version is the retirement - // blocker, which says what it means for the node. - debug!( - "Could not tell what {} is ({e}); treating it as neither removable nor \ - openable until it can be read", +/// Scan one shard directory into `result`. +fn scan_shard(dir: &Path, shard: u8, result: &mut ScanResult) -> Result<()> { + let entries = match std::fs::read_dir(dir) { + Ok(e) => e, + // A stray file named like a shard, or a directory removed between the two reads. + // Neither is fatal, and neither marks the shard as present. + Err(e) if matches!(e.kind(), ErrorKind::NotFound | ErrorKind::NotADirectory) => { + warn!( + "Chunk store: {} is not a shard directory ({e}); ignoring it", dir.display() ); - return RetirementMark::Unknown; + result.skipped = result.skipped.saturating_add(1); + return Ok(()); } - } - match dir.join(RETIRED_MARKER).try_exists() { - Ok(true) => RetirementMark::Present, - Ok(false) => RetirementMark::Absent, + // Anything else is a real fault: a permission problem, exhausted descriptors, or + // failing hardware. Opening with a shard's worth of keys silently missing would + // make the node under-claim in its published commitment and stop serving chunks + // it still holds and is answerable for, so refuse to open at all. Err(e) => { - debug!( - "Could not read the retirement mark in {} ({e}); treating it as neither \ - removable nor openable until it can be read", + return Err(Error::Storage(format!( + "Failed to enumerate shard {}: {e}. Refusing to open with an incomplete \ + key set.", dir.display() - ); - RetirementMark::Unknown - } - } -} - -/// Is this path a symbolic link, or something whose kind cannot be determined? -/// -/// Unknown counts as yes. Every caller is deciding whether it is safe to delete through -/// the path, and a question that cannot be answered is not a yes to that. -fn is_a_link(path: &Path) -> bool { - std::fs::symlink_metadata(path).map_or(true, |m| m.file_type().is_symlink()) -} - -/// Finish a removal a previous run did not, before anything tries to open the environment. -/// -/// The only thing that counts as evidence is the directory's own mark. An open that fails -/// is not: `open_legacy` queries free space, maps the file, takes a write transaction and -/// scans every key, so a full disk, a permission change, a mapping limit or a transient -/// I/O fault all look identical to corruption, and deleting on any of those would destroy -/// a perfectly good environment. -fn finish_interrupted_retirement(root_dir: &Path) -> LiveEnvironment { - let env = root_dir.join(LEGACY_ENV_DIR); - // Three answers, three branches. Asking only whether it may be removed and letting - // everything else fall through would put "cannot tell" back on the opening path, which - // is the whole failure this is three states to avoid: the mark check can fail for a - // moment and succeed the next, and the open in between would resurrect a store that - // really had been retired. - // - // Asked of the mark alone, with no separate "is it there" first. A `try_exists` that - // could not answer would have folded straight back into "nothing here" and skipped both - // branches below, which is the same fold one level up. The mark already tells the three - // apart: a path that is not there carries no mark and says so, and a path that cannot be - // reached at all says it cannot be reached. - // Asked once. Asking twice is asking two different questions: the answer can change - // between them, and a second answer of "cannot tell" after a first of "retired" fell - // through to opening the very directory the first answer said not to open. - let mark = retirement_mark(&env); - if mark == RetirementMark::Unknown { - error!( - "{} is under the live name and this node cannot tell whether it was retired. \ - It will NOT be opened and it will NOT be removed. The node serves from files \ - alone. Check that the directory and anything inside it can be read.", - env.display() - ); - return LiveEnvironment::None; - } - if mark.permits_removal() { - // Its own contents say it was retired, so whatever name it is wearing now, it is - // the remains of a removal that a power loss undid the rename of. - warn!( - "{} carries its own retirement mark, so it is what an interrupted removal left \ - behind rather than a live environment. Finishing that removal.", - env.display() - ); - // Renamed rather than deleted here, so the node can get on with starting: the - // deletion itself is detached below and can take minutes on a large store. Under a - // name nothing else is using, so a tombstone whose deletion is still running does - // not force a synchronous delete first. - let tombstone = free_tombstone_path(root_dir); - if let Err(e) = std::fs::rename(&env, &tombstone) { - error!( - "{} carries its own retirement mark but could not be moved aside: {e}. It \ - will NOT be opened: it says it has been retired, so it may be partly \ - deleted, and its chunks are in the file store. The node serves from files \ - alone and the next start tries again.", - env.display() - ); - sweep_retired_legacy(root_dir); - return LiveEnvironment::None; + ))) } + }; + if let Some(slot) = result.shards_present.get_mut(shard as usize) { + *slot = true; } - sweep_retired_legacy(root_dir); - LiveEnvironment::WhateverIsOnDisk -} -/// Whether the ordinary open may look at what is under the live environment name. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum LiveEnvironment { - /// Nothing is claiming it should not be opened. - WhateverIsOnDisk, - /// A directory under the live name says it has been retired, and could not be moved - /// out of the way. It must not be opened: a retired directory may be partly deleted, - /// and opening it would put its keys back into a commitment they have left. - None, -} - -fn sweep_retired_legacy(root_dir: &Path) { - let tombstones = retired_tombstones(root_dir); - if tombstones.is_empty() { - return; - } - // Flushed first, and only best effort is not good enough here for the same reason it - // was not good enough when the rename was made: deleting the contents of a directory - // whose new name may not have reached the disk is what turns a power loss into a - // resurrected, half-empty environment. If it cannot be flushed, leave them for a later - // start. It costs disk, not data. - if let Err(e) = crate::storage::file_store::fsync_path(root_dir) { - warn!( - "Leaving {} retired chunk environment(s) in place: {} could not be flushed \ - ({e}), so the rename that put them there may not be on disk yet.", - tombstones.len(), - root_dir.display() - ); - return; - } - for tombstone in tombstones { - // The name is not the evidence. Only the directory's own mark is: a crash between - // the rename and the mark leaves an intact environment sitting under the retired - // name, and deleting that because of what it is called would destroy every chunk - // in it. - let mark = retirement_mark(&tombstone); - if mark == RetirementMark::Unknown { - // Neither restored nor deleted. Restoring would put a directory that may be - // half-deleted back under the live name for the next start to open, and - // deleting would destroy an intact one. It costs disk until somebody looks, - // which is the right price for not knowing. - warn!( - "{} cannot be classified: this node cannot tell whether it carries a \ - retirement mark, so it will be neither restored nor deleted. Check that \ - the directory and anything inside it can be read.", - tombstone.display() - ); + for entry in entries { + let entry = + entry.map_err(|e| Error::Storage(format!("Failed to read {}: {e}", dir.display())))?; + let name = entry.file_name(); + let Some(name) = name.to_str() else { + result.skipped = result.skipped.saturating_add(1); + continue; + }; + if name.starts_with(TEMP_PREFIX) { + if sweep_temp(&entry.path()) { + result.swept_temps = result.swept_temps.saturating_add(1); + } continue; } - if mark.permits_removal() { - // The mark is re-established before anything is deleted on the strength of - // it. A retirement that failed part-way can leave one that was never flushed, - // and this is the pass that would otherwise act on it thirty seconds after - // the failure that said it would be left alone. - if let Err(e) = mark_directory_retired(&tombstone) { + let Some(key) = decode_chunk_name(name) else { + if name.len() == CHUNK_NAME_LEN && is_hex_any_case(name) { + // A case-folded twin of a real chunk name. On NTFS and default APFS the + // existence check in the write path folds onto it, so a paid write would + // be answered "already stored" and its bytes dropped. Move it aside. + quarantine_entry(&entry.path()); + } else { + warn!( + "Chunk store: ignoring non-chunk entry {name} in {}", + dir.display() + ); + } + result.skipped = result.skipped.saturating_add(1); + continue; + }; + // `file_type` comes from the directory entry itself on Linux and macOS and from + // the enumeration on Windows, so this is not the per-entry `stat` the scan + // deliberately avoids. A pipe, socket, device or directory wearing a chunk name + // must never enter the index: nothing downstream can read it, and it would sit in + // the published commitment forever. + match entry.file_type() { + Ok(kind) if kind.is_file() => {} + Ok(_) => { warn!( - "{} says it was retired but that could not be confirmed ({e}). \ - Leaving it.", - tombstone.display() + "Chunk store: {name} in {} is not a regular file; ignoring it", + dir.display() ); + result.skipped = result.skipped.saturating_add(1); continue; } - // Detached, so a node starting beside a large leftover directory serves - // immediately rather than waiting out a recursive delete before it opens its - // store. - delete_retired_directory(tombstone); + // Not the same as knowing it is not a file. Treating an unanswered question + // as a no would drop a real chunk from the index and from the commitment + // while its bytes sit on disk, and the node would not serve it again until + // some later restart happened to succeed. Fail the scan instead: an index + // that is missing keys must never be published as this node's key set. + Err(e) => { + return Err(Error::Storage(format!( + "Could not tell what {name} in {} is: {e}. Refusing to publish an \ + index that may be missing chunks.", + dir.display() + ))); + } + } + // A file in the wrong shard is unreachable through `chunk_path`, so indexing it + // would make the index claim a key the read path cannot find. + if shard_index(&key) != shard as usize { + warn!( + "Chunk store: {name} is filed under shard {shard:02x} but belongs in {:02x}; \ + ignoring it. Move it or delete it.", + shard_index(&key) + ); + result.skipped = result.skipped.saturating_add(1); continue; } - restore_unmarked_environment(root_dir, &tombstone); + result.keys.push(key); } + Ok(()) } -/// Put an intact environment back under its own name. +/// Remove one orphaned temp file. Returns whether it went. /// -/// An environment under the retired name with no mark inside it was renamed and then -/// interrupted before it could be marked. Nothing was deleted, so it is whole, and the -/// answer is to give it its name back and let the migration run again from the beginning: -/// every gate is re-derived, and a second retirement costs a pass, not data. -fn restore_unmarked_environment(root_dir: &Path, tombstone: &Path) { - // An empty one is what a deletion that removed the contents and the mark and then - // could not remove the directory leaves. There is nothing in it to restore, and - // putting it back under the live name would strand an empty path the node then tries - // to open. - if std::fs::read_dir(tombstone).is_ok_and(|mut entries| entries.next().is_none()) { - if let Err(e) = std::fs::remove_dir(tombstone) { - warn!("Could not remove the empty {}: {e}", tombstone.display()); - } - return; - } - let env = root_dir.join(LEGACY_ENV_DIR); - if env.try_exists().unwrap_or(true) { - // Both names are taken, so which one the node should serve is not this code's - // decision to make. - error!( - "{} and {} both exist, and {} carries no retirement mark, so it may hold \ - chunks. Neither has been touched. Move or remove one by hand: the node is \ - using {}.", - env.display(), - tombstone.display(), - tombstone.display(), - env.display() - ); - return; - } - match std::fs::rename(tombstone, &env) { +/// Always removed. The scan that calls this runs only after the store lock has been taken, +/// so by then any temp file is an interrupted write of a previous run and there is no other +/// process that could be writing it. This used to describe a second, gentler mode for the +/// unlocked case; there was never any such branch and there is no caller that would need +/// one. +fn sweep_temp(path: &Path) -> bool { + match std::fs::remove_file(path) { Ok(()) => { - let _ = crate::storage::file_store::fsync_path(root_dir); - warn!( - "{} was moved aside for retirement but never marked retired, so it is \ - intact. It has been restored to {} and the migration starts again.", - tombstone.display(), - env.display() - ); + debug!("Removed orphaned temporary file {}", path.display()); + true + } + Err(e) => { + debug!("Could not remove {}: {e}", path.display()); + false } - Err(e) => error!( - "{} carries no retirement mark, so it may hold chunks, but it could not be \ - restored to {}: {e}. It has not been deleted.", - tombstone.display(), - env.display() - ), } } -/// Every retired environment directory under `root_dir`. +/// Open a chunk file, refusing anything that is not a regular file. /// -/// More than one can be there: a node that retires, is restarted before the deletion -/// finishes, and somehow acquires another environment would leave the first behind. Each -/// is named so it cannot collide with the next. -fn retired_tombstones(root_dir: &Path) -> Vec { - let prefix = format!("{LEGACY_ENV_DIR}{RETIRED_SUFFIX}"); - let entries = match std::fs::read_dir(root_dir) { - Ok(entries) => entries, +/// `Ok(None)` means the file is not there. A named pipe wearing a valid chunk name would +/// otherwise block the opening thread forever: `open` on a FIFO with no writer does not +/// return, and enough of them would exhaust the blocking pool and stall every file and +/// database operation in the process. `O_NOFOLLOW` refuses a symlink for the same reason, +/// and both are checked on the handle rather than the path, so nothing can be swapped +/// underneath between the check and the open. +fn open_regular(path: &Path) -> Result> { + #[cfg(unix)] + let opened = { + use std::os::unix::fs::OpenOptionsExt; + OpenOptions::new() + .read(true) + .custom_flags(libc::O_NONBLOCK | libc::O_NOFOLLOW) + .open(path) + }; + #[cfg(not(unix))] + let opened = OpenOptions::new().read(true).open(path); + + let file = match opened { + Ok(f) => f, + Err(e) if e.kind() == ErrorKind::NotFound => return Ok(None), Err(e) => { - // Cannot tell. Not the same as nothing here, and the caller uses this to - // decide whether cleanup is finished, so answer with the one that keeps it - // looking rather than the one that declares victory. - warn!( - "Could not list {} to look for retired chunk environments: {e}", - root_dir.display() - ); - return vec![root_dir.join(&prefix)]; + return Err(Error::Storage(format!( + "Failed to open chunk file {}: {e}", + path.display() + ))) } }; - let mut found = Vec::new(); - for entry in entries { - match entry { - Ok(entry) => { - if entry - .file_name() - .to_str() - .is_some_and(|n| n.starts_with(&prefix)) - { - found.push(entry.path()); - } - } - // One unreadable entry is not evidence there is nothing here, and the caller - // uses this to decide whether cleanup is finished. Answer with the one that - // keeps it looking. - Err(e) => { - warn!( - "Could not read an entry of {} while looking for retired chunk \ - environments: {e}", - root_dir.display() - ); - found.push(root_dir.join(&prefix)); - } - } + let is_regular = file.metadata().is_ok_and(|m| m.file_type().is_file()); + if !is_regular { + return Err(Error::Storage(format!( + "{} is not a regular file; refusing to read it as a chunk", + path.display() + ))); } - found + Ok(Some(file)) } -/// A directory name to retire the environment under that nothing else is using. +/// Read a chunk file, refusing anything larger than a chunk can legitimately be. /// -/// A fixed name would collide with a tombstone whose deletion is still running, and -/// clearing that one first would put a synchronous recursive delete back on the path this -/// is trying to keep clear. -fn free_tombstone_path(root_dir: &Path) -> PathBuf { - let base = root_dir.join(format!("{LEGACY_ENV_DIR}{RETIRED_SUFFIX}")); - if !base.try_exists().unwrap_or(true) { - return base; - } - for n in 1..=MAX_TOMBSTONES { - let candidate = root_dir.join(format!("{LEGACY_ENV_DIR}{RETIRED_SUFFIX}.{n}")); - if !candidate.try_exists().unwrap_or(true) { - return candidate; - } +/// A corrupt, sparse, or locally planted file wearing a valid 64-hex name would +/// otherwise be read straight into memory, so a single bad entry could exhaust the node +/// during an ordinary GET or an audit response. +fn read_bounded(file: File, path: &Path) -> Result> { + let ceiling = MAX_CHUNK_SIZE as u64; + let mut buf = Vec::new(); + let read = file.take(ceiling + 1).read_to_end(&mut buf).map_err(|e| { + Error::Storage(format!("Failed to read chunk file {}: {e}", path.display())) + })?; + if read as u64 > ceiling { + return Err(Error::Storage(format!( + "Chunk file {} is larger than the {ceiling} byte maximum; refusing to read it", + path.display() + ))); } - // Every name taken, which means many retirements have been interrupted without their - // deletions finishing. Reuse the base: the rename fails, retirement defers, and the - // operator sees a directory full of them. - base + Ok(buf) } -/// Whether a legacy environment is on disk under `root_dir`. -/// -/// # Errors +/// Whether a Windows error is one a scanner or indexer holding a handle would produce. /// -/// Returns [`Error::Storage`] if the answer cannot be determined. `Path::exists` would -/// turn a permission problem into "absent", and a node that starts in file-only mode -/// beside a `chunks.mdb` holding every chunk it has stops serving all of them. -pub fn legacy_present(root_dir: &Path) -> Result { - let path = root_dir.join(LEGACY_ENV_DIR).join(LEGACY_DATA_FILE); - path.try_exists().map_err(|e| { - Error::Storage(format!( - "Cannot tell whether the legacy chunk environment {} exists: {e}. Refusing to \ - start rather than ignore it.", - path.display() - )) - }) +/// `ERROR_ACCESS_DENIED`, `ERROR_SHARING_VIOLATION`, `ERROR_LOCK_VIOLATION`. Every other +/// failure is deterministic and retrying it only burns a blocking thread. +fn is_windows_sharing_violation(e: &std::io::Error) -> bool { + matches!(e.raw_os_error(), Some(5 | 32 | 33)) } -/// How long to sleep after copying `bytes` to hold the copier to a rate ceiling. -fn throttle_delay(bytes: u64, mib_per_sec: u64) -> Option { - if mib_per_sec == 0 { - return None; - } - let per_sec = mib_per_sec.saturating_mul(1024 * 1024); - if per_sec == 0 { - return None; - } - let micros = bytes.saturating_mul(1_000_000) / per_sec; - if micros == 0 { - None - } else { - Some(Duration::from_micros(micros)) +/// Publish `temp_path` as `final_path`, retrying a transient sharing violation. +/// +/// On Windows an antivirus scanner or the search indexer can hold a handle to either +/// file for a few milliseconds after it is created, and `MoveFileEx` fails outright +/// rather than queueing. Retrying a bounded number of times turns that from a failed +/// write into a short pause. Every other error returns immediately. +fn rename_with_retry(temp_path: &Path, final_path: &Path) -> std::io::Result<()> { + let mut last = match std::fs::rename(temp_path, final_path) { + Ok(()) => return Ok(()), + Err(e) => e, + }; + if !cfg!(windows) || !is_windows_sharing_violation(&last) { + return Err(last); } -} - -/// Merge two ascending key sequences into one, dropping duplicates. -fn merge_sorted<'a, I>(sorted: &[XorName], other: I) -> Vec -where - I: Iterator, -{ - let other: Vec = other.copied().collect(); - let mut out = Vec::with_capacity(sorted.len() + other.len()); - let mut a = sorted.iter().copied().peekable(); - let mut b = other.into_iter().peekable(); - loop { - match (a.peek(), b.peek()) { - (Some(x), Some(y)) => match x.cmp(y) { - std::cmp::Ordering::Less => out.extend(a.next()), - std::cmp::Ordering::Greater => out.extend(b.next()), - std::cmp::Ordering::Equal => { - out.extend(a.next()); - let _ = b.next(); - } - }, - (Some(_), None) => out.extend(a.next()), - (None, Some(_)) => out.extend(b.next()), - (None, None) => break, + for attempt in 1..=RENAME_RETRY_ATTEMPTS { + std::thread::sleep(RENAME_RETRY_BACKOFF * attempt); + match std::fs::rename(temp_path, final_path) { + Ok(()) => return Ok(()), + Err(e) => last = e, } } - out + Err(last) } -#[cfg(test)] -#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] -mod tests { - use super::*; - use crate::storage::migration::{now_unix, rank_closest_first, MIN_RETIRE_DELAY_HOURS}; - use tempfile::TempDir; - - /// Everything currently legacy-only, as the set a test has "approved" for shedding. - fn approved_shed(store: &ChunkStore) -> BTreeSet { - store.legacy_only_keys().into_iter().collect() - } - - /// A token that is never cancelled, for tests that are not exercising shutdown. - fn never_cancelled() -> CancellationToken { - CancellationToken::new() - } - - /// Put a store through every gate a real node passes before it may retire. - /// - /// Deliberately not a shortcut around them: `retire_legacy` rechecks the whole set - /// itself, so a test that skipped them would exercise a path production never takes. - fn open_the_retirement_gate(store: &ChunkStore) { - store.commit_to_files().expect("commit"); - store.note_commitment_rebuilt(); - store.note_commitment_rebuilt(); - store.force_migration_state(|s| { - s.committed_at_unix = - Some(now_unix().saturating_sub(MIN_RETIRE_DELAY_HOURS * 3600 + 60)); - }); - } - - /// Content plus the address it hashes to. - fn addressed(seed: &str) -> (XorName, Vec) { - let content = format!("chunk-content-{seed}").into_bytes(); - (crate::client::compute_address(&content), content) - } - - fn test_config(dir: &TempDir) -> ChunkStoreConfig { - ChunkStoreConfig { - root_dir: dir.path().to_path_buf(), - ..ChunkStoreConfig::test_default() - } - } - - async fn open(dir: &TempDir) -> ChunkStore { - ChunkStore::new(test_config(dir)).await.expect("open store") - } - - /// Populate a legacy LMDB environment the way an existing node would have one, then - /// close it so the facade can adopt it. - async fn seed_legacy(dir: &TempDir, seeds: &[&str]) -> Vec { - let lmdb = LmdbStorage::new(LmdbStorageConfig { - root_dir: dir.path().to_path_buf(), - verify_on_read: true, - max_map_size: 0, - disk_reserve: 0, - }) - .await - .expect("open legacy"); - let mut keys = Vec::new(); - for seed in seeds { - let (addr, content) = addressed(seed); - lmdb.put(&addr, &content).await.expect("legacy put"); - keys.push(addr); - } - lmdb.wait_idle().await; - drop(lmdb); - keys - } - - #[tokio::test] - async fn a_fresh_node_never_creates_a_legacy_environment() { - let dir = TempDir::new().expect("temp dir"); - let store = open(&dir).await; - - assert!(!store.has_legacy()); - assert_eq!(store.migration_phase(), MigrationPhase::FilesOnly); - assert!(!dir.path().join(LEGACY_ENV_DIR).exists()); - - let (addr, content) = addressed("fresh"); - assert!(store.put(&addr, &content).await.expect("put")); - assert_eq!( - store.get(&addr).await.expect("get").expect("present"), - content - ); - } - - #[tokio::test] - async fn an_existing_legacy_store_is_adopted_and_served() { - let dir = TempDir::new().expect("temp dir"); - let keys = seed_legacy(&dir, &["a", "b", "c"]).await; - let store = open(&dir).await; - - assert!(store.has_legacy()); - assert_eq!(store.migration_phase(), MigrationPhase::Bridging); - assert_eq!(store.current_chunks().expect("count"), 3); - for key in &keys { - assert!(store.exists(key).expect("exists"), "union must see it"); - assert!(store.get(key).await.expect("get").is_some()); - } - } - - #[tokio::test] - async fn the_union_key_set_is_sorted_and_free_of_duplicates() { - let dir = TempDir::new().expect("temp dir"); - let mut expected = seed_legacy(&dir, &["u1", "u2", "u3", "u4"]).await; - let store = open(&dir).await; - - // One chunk written now lives in both backings, and must be counted once. - let (addr, content) = addressed("u2"); - assert!(expected.contains(&addr)); - assert!(!store.put(&addr, &content).await.expect("put")); - - let (fresh, fresh_content) = addressed("u5"); - store.put(&fresh, &fresh_content).await.expect("put"); - expected.push(fresh); - expected.sort_unstable(); - - let keys = store.all_keys().await.expect("all_keys"); - assert_eq!(keys, expected); - assert_eq!(store.current_chunks().expect("count"), 5); - } - - /// The legacy environment takes no new disk during the bridge. - /// - /// Both stores sit on one disk, each measures the same free space, and neither knows - /// what the other is about to spend. A chunk written to both could be admitted twice - /// against one lot of headroom, and enough of them could cross the reserve together - /// and fill the volume this migration exists to free. - /// - /// So the environment is pinned to what it already occupies. It still takes the - /// rollback copy when it has room of its own, which on a real node it usually does: - /// this migration exists because deleting millions of chunks left the free list full - /// and returned nothing to the filesystem. What it will not do is grow. - #[tokio::test] - async fn the_legacy_environment_takes_no_new_disk_during_the_bridge() { - let dir = TempDir::new().expect("temp dir"); - seed_legacy(&dir, &["seed"]).await; - let store = open(&dir).await; - - let data_file = dir.path().join(LEGACY_ENV_DIR).join(LEGACY_DATA_FILE); - let before = std::fs::metadata(&data_file).expect("meta").len(); - - for seed in ["dual-1", "dual-2", "dual-3", "dual-4"] { - let (addr, content) = addressed(seed); - assert!(store.put(&addr, &content).await.expect("put")); - assert!( - store.exists(&addr).expect("exists"), - "the file store is the one that has to have it" - ); +/// Write `payload` and publish it as `final_path`, replacing whatever is there. +/// +/// Success here means the bytes are durable, not merely written. The repair path this +/// serves runs during the pre-retirement pass, where a chunk that fails to match its +/// address is rewritten from the legacy store and the legacy store is then deleted. A +/// replacement that a power loss can undo would leave that chunk with the wrong bytes and +/// no other copy. +fn write_and_replace( + temp_path: &Path, + final_path: &Path, + payload: &[u8], + shard: &Path, +) -> Result<()> { + // Unix: an intra-directory rename is atomic, so a reader sees the old content or the + // new one and never an absence, and the directory flush is what makes it durable. + #[cfg(unix)] + { + write_temp(temp_path, payload)?; + if let Err(e) = rename_with_retry(temp_path, final_path) { + let _ = std::fs::remove_file(temp_path); + return Err(Error::Storage(format!( + "Failed to replace chunk {}: {e}", + final_path.display() + ))); } - store.wait_idle().await; - - let after = std::fs::metadata(&data_file).expect("meta").len(); - assert_eq!( - after, before, - "the environment must not claim disk the file store is also counting on" - ); + fsync_dir(shard).map_err(|e| { + Error::Storage(format!( + "Replaced {} but could not flush {}: {e}. Not reporting the repair as \ + done, because a rewrite that is not durable must not authorise deleting \ + the copy it was rewritten from.", + final_path.display(), + shard.display() + )) + })?; + Ok(()) } - - #[tokio::test] - async fn the_copier_moves_keys_into_files_and_is_resumable() { - let dir = TempDir::new().expect("temp dir"); - let keys = seed_legacy(&dir, &["c1", "c2", "c3", "c4", "c5"]).await; - let store = open(&dir).await; - assert_eq!(store.legacy_only_keys().len(), 5); - - let first = store - .copy_batch(&keys[..2], 0, 0, &never_cancelled()) - .await - .expect("copy first batch"); - assert_eq!(first.copied, 2); - assert_eq!(store.legacy_only_keys().len(), 3); - store.wait_idle().await; - drop(store); - - // A restart re-derives what is left from the filesystem: no progress file to - // corrupt, and no work repeated. - let store = open(&dir).await; - assert_eq!(store.legacy_only_keys().len(), 3); - let rest = store - .copy_batch(&keys, 0, 0, &never_cancelled()) - .await - .expect("copy rest"); - assert_eq!(rest.copied, 3); - assert!(store.legacy_only_keys().is_empty()); - assert_eq!(store.current_chunks().expect("count"), 5); + // Everywhere else, Windows included: there is no way to flush a directory through the + // standard library, so a rename cannot be shown to be durable at return. Overwriting + // the existing file changes no directory entry at all, and `sync_all` (FlushFileBuffers + // on Windows) is documented to flush the file's data, so a successful return is + // durable under a documented contract. + // + // The cost is that this is not atomic: a crash part-way leaves the file holding a mix + // of old and new bytes. + // + // That used to be justified by the legacy store still being there to repair from, which + // it no longer is. The argument now is narrower and does not depend on a second copy: + // every caller reaches this only after a read has proven the bytes under that name + // wrong. A crash part-way therefore leaves wrong bytes where wrong bytes already were, + // which is not a loss, and the next verified read finds them and repairs again. What it + // is NOT safe for is replacing bytes that were good, so this must not be reached on any + // path that has not established otherwise. In this crate that holds: the two + // `StoredBytes::Wrong` arms get there from a read that hashed and disagreed, and + // `holds_verified` gets there from a read that returned bytes which were not the + // caller's. The public entry point makes no such check and says so. + // + // A temporary and a rename would make it atomic, at the cost of a directory entry + // change that cannot be flushed here. That trade is worth revisiting on a platform + // where it can actually be tested; it is not worth making blind. + #[cfg(not(unix))] + { + let _ = temp_path; + let _ = shard; + let mut file = OpenOptions::new() + .write(true) + .truncate(true) + .open(final_path) + .map_err(|e| { + Error::Storage(format!( + "Failed to open {} for replacement: {e}", + final_path.display() + )) + })?; + file.write_all(payload).map_err(|e| { + Error::Storage(format!("Failed to rewrite {}: {e}", final_path.display())) + })?; + file.sync_all().map_err(|e| { + Error::Storage(format!( + "Rewrote {} but could not flush it: {e}. Not reporting the repair as \ + done, because a rewrite that is not durable must not authorise deleting \ + the copy it was rewritten from.", + final_path.display() + )) + })?; + Ok(()) } +} - #[tokio::test] - async fn a_delete_reaches_both_stores() { - let dir = TempDir::new().expect("temp dir"); - let keys = seed_legacy(&dir, &["d1", "d2"]).await; - let store = open(&dir).await; - - let target = keys.first().copied().expect("a key"); - assert!(store.delete(&target).await.expect("delete")); - assert!(!store.exists(&target).expect("exists")); - assert!(store.get(&target).await.expect("get").is_none()); - assert_eq!(store.current_chunks().expect("count"), 1); - - // And it stays gone across a restart, which is what proves it left the legacy - // environment too rather than only the union view. - store.wait_idle().await; - drop(store); - let store = open(&dir).await; - assert!(!store.exists(&target).expect("exists")); +/// Create `temp_path`, write `payload` into it, and flush it. +/// +/// Flushed before any rename. On ext4 `auto_da_alloc` only orders the data before the +/// rename's own commit; it does not make the data durable, and btrfs has been observed +/// reordering. A name must never become visible on bytes that are not on the platter. +fn write_temp(temp_path: &Path, payload: &[u8]) -> Result<()> { + let mut f = OpenOptions::new() + .write(true) + .create_new(true) + .open(temp_path) + .map_err(|e| { + Error::Storage(format!( + "Failed to create temporary file {}: {e}", + temp_path.display() + )) + })?; + if let Err(e) = f.write_all(payload) { + let _ = std::fs::remove_file(temp_path); + return Err(Error::Storage(format!( + "Failed to write {}: {e}", + temp_path.display() + ))); } - - #[tokio::test] - async fn the_copier_does_not_resurrect_a_deleted_chunk() { - let dir = TempDir::new().expect("temp dir"); - let keys = seed_legacy(&dir, &["r1", "r2"]).await; - let store = open(&dir).await; - - let target = keys.first().copied().expect("a key"); - store.delete(&target).await.expect("delete"); - - let report = store - .copy_batch(&keys, 0, 0, &never_cancelled()) - .await - .expect("copy"); - assert_eq!(report.copied, 1, "only the surviving chunk may be copied"); - assert!(!store.exists(&target).expect("exists")); + if let Err(e) = f.sync_all() { + let _ = std::fs::remove_file(temp_path); + return Err(Error::Storage(format!( + "Failed to flush {}: {e}", + temp_path.display() + ))); } + Ok(()) +} - #[tokio::test] - async fn committing_narrows_the_commitment_but_not_what_is_served() { - let dir = TempDir::new().expect("temp dir"); - let keys = seed_legacy(&dir, &["k1", "k2", "k3"]).await; - let store = open(&dir).await; - - // Copy one, leave two behind as if the disk had run out. - store - .copy_batch(&keys[..1], 0, 0, &never_cancelled()) - .await - .expect("copy"); - - // While bridging, the node still claims everything it can serve. - assert_eq!( - store.committable_keys().await.expect("committable").len(), - 3 - ); +/// Write `payload` and publish it under `final_path`. +/// +/// The temp lives in the destination directory, so the publish is an intra-directory +/// rename: atomic on every filesystem we support, and needing only that one directory +/// Put `payload` on disk as `final_path`, durably. +/// +/// Returns [`PutOutcome::Duplicate`] when the name is already taken. The name is a hash +/// of the content, so that is not treated as proof the bytes are right: the caller +/// re-reads and verifies them. +#[cfg(unix)] +fn publish( + temp_path: &Path, + final_path: &Path, + payload: &[u8], + shard: &Path, +) -> std::result::Result { + // On Unix nothing is ever created under the final name by a failing path: the bytes go + // to a temporary and only a successful rename gives them the real name. So every + // failure here leaves the name as it found it. + publish_via_rename(temp_path, final_path, payload, shard) + .map_err(PublishFailed::nothing_written) +} - store.commit_to_files().expect("commit"); - assert_eq!(store.migration_phase(), MigrationPhase::Committed); - assert_eq!(store.migration_state().shed_key_count, 2); +/// Put `payload` on disk as `final_path`, durably. See [`publish_in_place`] for why this +/// takes a different route off Unix. +#[cfg(not(unix))] +fn publish( + temp_path: &Path, + final_path: &Path, + payload: &[u8], + shard: &Path, +) -> std::result::Result { + let _ = temp_path; + let _ = shard; + publish_in_place(final_path, payload) +} - // It now claims only what it will keep... - assert_eq!( - store.committable_keys().await.expect("committable").len(), - 1 - ); - // ...while still serving everything it ever claimed. - assert_eq!(store.all_keys().await.expect("all_keys").len(), 3); - for key in &keys { - assert!(store.get(key).await.expect("get").is_some()); +/// Create the chunk under its final name and flush it. Everywhere but Unix. +/// +/// There is no way to flush a directory through the standard library, and Microsoft does +/// not document `MoveFileEx` as durable at return unless it is called with +/// `MOVEFILE_WRITE_THROUGH`, which std does not use. So off Unix a rename cannot be +/// relied on to have reached the disk before the legacy store is deleted. +/// +/// Creating the file under its final name sidesteps the rename entirely. Microsoft +/// documents that creation metadata is cached and that `FlushFileBuffers`, which +/// `sync_all` calls on Windows, is the way to flush it. So a successful create, write and +/// flush is a durable publication under a documented contract, with no directory flush +/// and no rename involved. +/// +/// The cost is that a crash mid-write leaves a partial file wearing a real chunk name. +/// That is why a duplicate re-reads and verifies rather than trusting the name, and why +/// the pre-retirement pass re-hashes everything before anything is deleted. +#[cfg(not(unix))] +fn publish_in_place( + final_path: &Path, + payload: &[u8], +) -> std::result::Result { + // Test-only, and here rather than after the write so that it means the same thing on + // both platforms: the file half of a dual write has not happened yet. On Unix the + // equivalent point is the temporary file written and the rename not yet made, which is + // also before the chunk's name exists on disk. Stopping after the write instead would + // put the file under its real name already, so a crash there is not between the two + // halves at all, and it could not demonstrate anything about the missing flush either: + // killing a process does not empty the page cache, so the bytes are still there to be + // read. Only losing power loses them, which no test that kills a process can stage. + #[cfg(any(test, feature = "test-utils"))] + halt_here_if_asked(HALT_BEFORE_PUBLISH, final_path); + let mut file = match OpenOptions::new() + .write(true) + .create_new(true) + .open(final_path) + { + Ok(f) => f, + // Someone got there first. Immutable content under a content-addressed name, so + // the caller verifies what is already there rather than assuming it is right. + Err(e) if e.kind() == ErrorKind::AlreadyExists => return Ok(PutOutcome::Duplicate), + Err(e) => { + // Nothing was created, so nothing was spent. + return Err(PublishFailed::nothing_written(Error::Storage(format!( + "Failed to create chunk {}: {e}", + final_path.display() + )))); } + }; + if let Err(e) = file.write_all(payload) { + drop(file); + // Taken back if it can be. Whether it could is what the caller needs: the file + // was created by this call, so if it is still there the space is spent. + let left_behind = std::fs::remove_file(final_path).is_err(); + return Err(PublishFailed { + error: Error::Storage(format!("Failed to write {}: {e}", final_path.display())), + left_behind, + }); } - - #[tokio::test] - async fn retirement_is_refused_until_every_gate_is_satisfied() { - let dir = TempDir::new().expect("temp dir"); - let keys = seed_legacy(&dir, &["g1", "g2"]).await; - let mut config = test_config(&dir); - config.migration.retire_legacy = true; - let store = ChunkStore::new(config).await.expect("open"); - - // Still bridging. - assert!(store - .retirement_blocker(|_| false) - .expect("blocked") - .contains("Bridging")); - - store - .copy_batch(&keys, 0, 0, &never_cancelled()) - .await - .expect("copy"); - store.commit_to_files().expect("commit"); - - // No commitment rebuild observed yet. - assert!(store - .retirement_blocker(|_| false) - .expect("blocked") - .contains("commitment rebuilds")); - - store.note_commitment_rebuilt(); - store.note_commitment_rebuilt(); - - // The retention delay has not elapsed. - assert!(store - .retirement_blocker(|_| false) - .expect("blocked") - .contains("retirement delay")); - - store.force_migration_state(|s| { - s.committed_at_unix = - Some(now_unix().saturating_sub(MIN_RETIRE_DELAY_HOURS * 3600 + 60)); - }); - assert!(store.retirement_blocker(|_| false).is_none()); - } - - #[tokio::test] - async fn a_chunk_still_answerable_vetoes_retirement() { - let dir = TempDir::new().expect("temp dir"); - seed_legacy(&dir, &["h1", "h2"]).await; - let mut config = test_config(&dir); - config.migration.retire_legacy = true; - let store = ChunkStore::new(config).await.expect("open"); - - // Shed both, as a node short of disk would. - store.commit_to_files().expect("commit"); - store.note_commitment_rebuilt(); - store.note_commitment_rebuilt(); - store.force_migration_state(|s| { - s.committed_at_unix = - Some(now_unix().saturating_sub(MIN_RETIRE_DELAY_HOURS * 3600 + 60)); - }); - - // The pruner's existing retention contract, reused verbatim: a key the node - // could still be challenged on keeps its last local copy. - assert!(store - .retirement_blocker(|_| true) - .expect("blocked") - .contains("still answerable")); - assert!(store.retirement_blocker(|_| false).is_none()); - } - - /// The stock configuration retires, with no environment variable and no operator step. - /// - /// This is the property the whole release rests on. Deleting `chunks.mdb` is the only - /// step that returns disk: LMDB never gives freed pages back, which is why the fleet - /// deleted millions of chunks and recovered nothing. A build that shipped with this - /// off would migrate every node and reclaim not one byte. - #[tokio::test] - async fn the_shipped_configuration_retires_without_an_operator_setting_anything() { - // Asked of the configuration a node actually builds, not of the constant behind - // it, so neither the constant nor a serde default nor the `Default` impl can turn - // retirement off without this failing. - assert!( - crate::storage::MigrationConfig::default().retire_legacy, - "this release must delete the legacy environment, or it frees no disk" - ); - - let dir = TempDir::new().expect("temp dir"); - let keys = seed_legacy(&dir, &["shipped"]).await; - let store = open(&dir).await; - store - .copy_batch(&keys, 0, 0, &never_cancelled()) - .await - .expect("copy"); - open_the_retirement_gate(&store); - assert!( - store.retirement_blocker(|_| false).is_none(), - "with every gate met, the shipped configuration must not refuse to retire" - ); - } - - /// Retirement waits for a read that is already running. - /// - /// The window this closes: a verifying read finds rotted bytes, throws the file away, - /// and has not yet reached the legacy copy that would replace it. If retirement ran in - /// that gap it would delete the only remaining copy. Holding the barrier shared for - /// the whole read, and exclusively for the removal, is what makes that impossible. - #[tokio::test] - async fn retirement_waits_for_a_read_that_is_already_running() { - let dir = TempDir::new().expect("temp dir"); - let keys = seed_legacy(&dir, &["held"]).await; - let store = Arc::new(open(&dir).await); - store - .copy_batch(&keys, 0, 0, &never_cancelled()) - .await - .expect("copy"); - open_the_retirement_gate(&store); - let proof = store - .verify_before_retire(0, &never_cancelled()) - .await - .expect("verify"); - - // Stand in for a read that has started and not finished. - let reading = store.retirement.read().await; - - let retiring = { - let store = Arc::clone(&store); - let approved = approved_shed(&store); - tokio::spawn(async move { - store - .retire_legacy(&proof, &|_: &XorName| false, &approved) - .await - }) - }; - - // It must not have got anywhere. Given a generous window rather than a tight one, - // so this fails on the behaviour rather than on scheduling luck. - tokio::time::sleep(Duration::from_millis(200)).await; - assert!( - !retiring.is_finished(), - "retirement removed the legacy environment while a read was still running" - ); - assert!( - store.has_legacy(), - "the legacy environment went while a read was still running" - ); - - drop(reading); - let freed = retiring.await.expect("join").expect("retire"); - assert!(freed > 0, "retirement should have freed the environment"); - assert!(!store.has_legacy()); - } - - /// A good copy is never turned away because a damaged one wears its name. - /// - /// Two shapes of damage, because they are caught differently: a short file, which is - /// what an interrupted create leaves on a platform that writes under the final name, - /// and a full-length file with wrong bytes, which is what rot leaves. Answering - /// "already have it" to either discards the copy that would fix it, and nothing offers - /// it again. - #[tokio::test] - async fn a_damaged_chunk_is_repaired_from_the_copy_being_offered() { - let dir = TempDir::new().expect("temp dir"); - let store = open(&dir).await; - let (addr, content) = addressed("repairable-by-offer"); - store.put(&addr, &content).await.expect("put"); - - // Intact: the offer is correctly refused. - assert!(store.holds_verified(&addr, &content).await); - - // Truncated. - let path = dir - .path() - .join("chunks") - .join(format!("{:02x}", addr.last().copied().unwrap_or(0))) - .join(hex::encode(addr)); - std::fs::write(&path, &content[..content.len() / 2]).expect("truncate"); - assert!( - store.holds_verified(&addr, &content).await, - "a short file must be replaced from the offered copy, not left in place" - ); - assert_eq!( - store.get(&addr).await.expect("get").expect("present"), - content - ); - - // Same length, wrong bytes. - let rotted = vec![b'x'; content.len()]; - assert_ne!(rotted, content); - std::fs::write(&path, &rotted).expect("rot"); - assert!( - store.holds_verified(&addr, &content).await, - "a rotted file must be replaced from the offered copy" - ); - assert_eq!( - store.get(&addr).await.expect("get").expect("present"), - content - ); - } - - /// A chunk held only in the legacy environment is checked, not assumed good. - /// - /// The bytes in there can be wrong too, and when the copier finds that out it drops - /// the key from the union view. Having turned the good copy away on the strength of - /// the key being present, the node would then hold nothing at all. - #[tokio::test] - async fn a_legacy_only_chunk_with_wrong_bytes_is_replaced_by_the_offer() { - let dir = TempDir::new().expect("temp dir"); - let keys = seed_legacy(&dir, &["legacy-side"]).await; - let key = *keys.first().expect("one key"); - let content = format!("chunk-content-{}", "legacy-side").into_bytes(); - let store = open(&dir).await; - assert!(store.legacy_only_keys().contains(&key)); - - // Intact: the offer is correctly refused. - assert!(store.holds_verified(&key, &content).await); - - // Wreck the legacy copy underneath, leaving the key in the union view. - let legacy = store.legacy().expect("legacy"); - legacy.lmdb.delete(&key).await.expect("delete"); - assert!( - store.holds_verified(&key, &content).await, - "with no readable legacy copy the offered bytes must be taken, not refused" - ); - assert_eq!( - store.get(&key).await.expect("get").expect("present"), - content - ); - assert!( - !store.legacy_only_keys().contains(&key), - "and the key must leave the legacy-only set now that a file holds it" - ); - } - - /// A chunk this node does not have is not claimed as held. - #[tokio::test] - async fn a_chunk_this_node_does_not_have_is_not_claimed() { - let dir = TempDir::new().expect("temp dir"); - let store = open(&dir).await; - let (addr, content) = addressed("never-stored"); - assert!(!store.holds_verified(&addr, &content).await); + if let Err(e) = file.sync_all() { + drop(file); + // Taken back if it can be. Whether it could is what the caller needs: the file + // was created by this call, so if it is still there the space is spent. + let left_behind = std::fs::remove_file(final_path).is_err(); + return Err(PublishFailed { + error: Error::Storage(format!("Failed to flush {}: {e}", final_path.display())), + left_behind, + }); } + Ok(PutOutcome::New) +} - /// An environment is never deleted because it failed to open. - /// - /// Opening is not a corruption test. It queries free space, maps the file, takes a - /// write transaction and scans every key, so a full disk, a permission change or a - /// transient fault all look identical to corruption. The only thing that counts is the - /// directory's own mark. - #[tokio::test] - async fn an_unmarked_environment_is_kept_however_badly_it_reads() { - let dir = TempDir::new().expect("temp dir"); - seed_legacy(&dir, &["unmarked"]).await; - let env = dir.path().join(LEGACY_ENV_DIR); - assert_eq!(retirement_mark(&env), RetirementMark::Absent); +/// Write a temp beside the target and rename it into place. Unix only. +/// +/// Places the bytes and nothing more. Making the name durable is +/// [`flush_publication`]'s job, kept separate so a caller can tell a publish that spent no +/// space from one that spent it and could not be reported. +#[cfg(unix)] +fn publish_via_rename( + temp_path: &Path, + final_path: &Path, + payload: &[u8], + _shard: &Path, +) -> Result { + // Content is immutable and the name is its hash, so an existing file already holds + // exactly these bytes. Skipping the write is both cheaper and safer than replacing + // it: on Windows a rename over a file another thread has open fails outright. + // + // The caller flushes either way. A name that is already there is not proof it is + // durable: + // the write that put it there may have been this store's own previous attempt, whose + // rename landed and whose directory flush then failed. That attempt returned an + // error, so nothing was retired on the strength of it, but if this call reported a + // durable duplicate without flushing, the retry would silently launder an unflushed + // rename into a copy that authorises deleting the last other one. + let outcome = if final_path.exists() { + PutOutcome::Duplicate + } else { + write_temp(temp_path, payload)?; + // Test-only: the one moment a complete chunk exists on disk under a name nothing + // looks for. A crash test needs to die at a named point rather than wherever a + // sleep in another process happened to land. + #[cfg(any(test, feature = "test-utils"))] + halt_here_if_asked(HALT_BEFORE_PUBLISH, temp_path); + match rename_with_retry(temp_path, final_path) { + Ok(()) => PutOutcome::New, + Err(e) => { + let _ = std::fs::remove_file(temp_path); + // Another writer of the same address won the race, or the destination was + // open. Either way the bytes are already published. + if !final_path.exists() { + return Err(Error::Storage(format!( + "Failed to publish chunk {}: {e}", + final_path.display() + ))); + } + PutOutcome::Duplicate + } + } + }; - finish_interrupted_retirement(dir.path()); - assert!( - env.exists(), - "an environment carrying no retirement mark must never be removed" - ); - } + Ok(outcome) +} - /// A directory carrying its own retirement mark is finished off, whatever it is named. - /// - /// This is the case the mark exists for. Off Unix the rename that moves the - /// environment aside cannot be shown to be durable, so a power loss can bring it back - /// under its old name with its contents already deleted. Without the mark the node - /// would refuse to start on it forever; with it, the directory says what it is. - #[tokio::test] - async fn a_directory_that_says_it_was_retired_is_removed_under_any_name() { - let dir = TempDir::new().expect("temp dir"); - let keys = seed_legacy(&dir, &["reverted"]).await; - { - let store = open(&dir).await; - store - .copy_batch(&keys, 0, 0, &never_cancelled()) - .await - .expect("copy"); - } - // What a reverted rename leaves: the old name, the retirement mark inside it. - let env = dir.path().join(LEGACY_ENV_DIR); - mark_directory_retired(&env).expect("mark"); +/// A publish that failed, and whether it left its bytes on the disk. +/// +/// The second half is the point. A failure before anything was created has spent nothing; +/// one that created the file and then could not remove it again has spent the space, and +/// whoever is accounting for free space has to know which happened. Only the code that did +/// the creating can say. +struct PublishFailed { + error: Error, + left_behind: bool, +} - let store = open(&dir).await; - assert!( - !store.has_legacy(), - "the remains of an interrupted removal must not be adopted" - ); - assert!(!env.exists(), "and the next start must finish the removal"); - // Every chunk is still served, from the file store. - for key in &keys { - assert!(store.get(key).await.expect("get").is_some()); +impl PublishFailed { + /// A failure that created nothing. + fn nothing_written(error: Error) -> Self { + Self { + error, + left_behind: false, } } +} - /// The mark goes inside the directory, so a rename cannot separate them. - /// - /// A mark beside the environment would have to be cancelled when a retirement is - /// abandoned, cancellation can fail or be lost, and a stale one would then authorise - /// deleting an environment that had since taken a chunk. - #[test] - fn the_retirement_mark_travels_with_the_directory() { - let dir = TempDir::new().expect("temp dir"); - let original = dir.path().join("chunks.mdb"); - std::fs::create_dir_all(&original).expect("mkdir"); - mark_directory_retired(&original).expect("mark"); - assert_eq!(retirement_mark(&original), RetirementMark::Present); +/// Make a publication durable by flushing the directory its name lives in. +/// +/// Separate from placing the bytes, because the caller has to tell the two failures apart. +/// A publish that fails before the bytes land has spent nothing; one that fails here has +/// spent the space and must not be reported as stored, so whoever is accounting for free +/// space has to charge it while whoever is accounting for chunks must not count it. +/// +/// NOT best effort. The directory flush is what makes the rename durable, and a copy +/// reported successful is what authorises deleting the only other copy. Swallowing the +/// failure would let a power loss discard the directory entry after the legacy store had +/// already been removed. +/// +/// # Errors +/// +/// Returns [`Error::Storage`] if the directory cannot be flushed. +#[cfg(unix)] +fn flush_publication(final_path: &Path, shard: &Path) -> Result<()> { + fsync_dir(shard).map_err(|e| { + Error::Storage(format!( + "Published {} but could not flush {}: {e}. Not reporting this chunk as stored, \ + because a copy that is not durable must not authorise deleting another.", + final_path.display(), + shard.display() + )) + }) +} - let renamed = dir.path().join("chunks.mdb.retired"); - std::fs::rename(&original, &renamed).expect("rename"); - assert!( - retirement_mark(&renamed) == RetirementMark::Present, - "the mark must survive the rename it exists to outlive" - ); - // And back again, which is what a power loss undoing the rename looks like. - std::fs::rename(&renamed, &original).expect("rename back"); - assert_eq!(retirement_mark(&original), RetirementMark::Present); - } +/// Nothing to do off Unix, where the chunk is created under its final name and flushed +/// with `sync_all`, which is documented to carry its creation metadata with it, and where +/// there is no way to flush a directory at all. +#[cfg(not(unix))] +fn flush_publication(_final_path: &Path, _shard: &Path) -> Result<()> { + Ok(()) +} + +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +mod tests { + use super::*; + use std::collections::HashSet; - /// Losing the handle to an environment that is still there is not completion. + /// A directory flush that fails must say so. /// - /// A rename that failed and then could not be reopened leaves the directory on disk - /// with no way to read it. Treating the missing handle as "nothing left to migrate" - /// would have the driver log the migration finished over a store still holding chunks - /// nothing else can serve. - #[tokio::test] - async fn a_lost_handle_beside_a_live_environment_blocks_retirement() { + /// The quiet version of this function is only used where the answer does not change + /// what happens next. On the publish path it does. + #[cfg(unix)] + #[test] + fn flushing_a_directory_that_is_not_there_reports_the_failure() { let dir = TempDir::new().expect("temp dir"); - seed_legacy(&dir, &["orphaned"]).await; - let store = open(&dir).await; - assert!(store.has_legacy()); - - // Stand in for a failed rename followed by a failed reopen. - *store.legacy.write() = None; - assert!(!store.has_legacy()); - assert!(dir.path().join(LEGACY_ENV_DIR).exists()); - - let blocker = store - .retirement_blocker(|_| false) - .expect("a live environment with no handle must block"); - assert!( - blocker.contains("no handle"), - "the reason must name the actual problem, got: {blocker}" - ); + assert!(fsync_dir(dir.path()).is_ok()); + assert!(fsync_dir(&dir.path().join("no-such-shard")).is_err()); } - /// A node that still holds its store open is gated too. + /// A chunk whose directory entry was never flushed is not reported as stored. /// - /// The classification used to be asked only when there was no handle, which meant the - /// ordinary path never asked it: a node holding its environment open went through every - /// gate, renamed the directory aside and deleted it, whatever the mark said or failed to - /// say. Retirement deletes the last other copy of these chunks, so it is not a question - /// to skip because a different question already had an answer. + /// This is the whole safety argument for retirement: the legacy store is deleted + /// because every chunk was copied durably. A published file whose directory flush + /// failed can vanish on power loss, so counting it as copied would lose data. The + /// file staying on disk afterwards is fine, the next pass republishes it. #[cfg(unix)] - #[tokio::test] - async fn an_environment_that_cannot_be_classified_blocks_retirement_even_with_a_handle() { + #[test] + fn a_publish_whose_directory_flush_fails_is_not_reported_as_stored() { let dir = TempDir::new().expect("temp dir"); - seed_legacy(&dir, &["still-open"]).await; - let store = open(&dir).await; - assert!( - store.has_legacy(), - "this one keeps its handle, deliberately" - ); - // Whatever else is in the way at this point, it is not this. Compared rather than - // required to be nothing, because the other gates have their own tests and their - // own reasons to be unmet here. - let before = store.retirement_blocker(|_| false).unwrap_or_default(); + let temp_path = dir.path().join("chunk.tmp"); + let final_path = dir.path().join("chunk"); + let unflushable = dir.path().join("shard-that-does-not-exist"); + + // Asserted in two steps, not chained. Chaining them means a regression in placing + // the bytes also produces an error, and the test passes without the flush ever + // being reached: it would be checking that something went wrong rather than that + // this went wrong. + let placed = publish_via_rename(&temp_path, &final_path, b"payload", &unflushable); assert!( - !before.contains("already retired"), - "a readable environment must not be blocked for being unclassifiable: {before}" + placed.is_ok(), + "the bytes must be placed before this can be about the flush: {:?}", + placed.err() ); + let outcome = flush_publication(&final_path, &unflushable); - make_the_mark_unreadable(&dir.path().join(LEGACY_ENV_DIR)); - - let blocker = store - .retirement_blocker(|_| false) - .expect("an environment that cannot be classified must block retirement"); assert!( - blocker.contains("already retired"), - "the reason must name the actual problem, got: {blocker}" + outcome.is_err(), + "an unflushed publication must not be reported as stored" ); assert!( - store.legacy_cannot_be_classified(), - "and the driver must see it as work only a person can finish, so that it \ - gives the shared volume back" + !temp_path.exists(), + "the temp file must not be left behind either way" ); } - /// A directory under the retired name with no mark inside it is an intact store. - /// - /// It got that name from a rename, and the rename happens after every gate; the mark - /// is written straight afterwards. A crash in between leaves a whole environment - /// wearing a name that says otherwise, and deleting it because of what it is called - /// would destroy every chunk in it. It is put back instead. - #[tokio::test] - async fn an_unmarked_retired_directory_is_restored_rather_than_deleted() { - let dir = TempDir::new().expect("temp dir"); - let keys = seed_legacy(&dir, &["not-really-retired"]).await; - let env = dir.path().join(LEGACY_ENV_DIR); - let tombstone = dir.path().join(format!("{LEGACY_ENV_DIR}{RETIRED_SUFFIX}")); - std::fs::rename(&env, &tombstone).expect("rename"); - assert_eq!(retirement_mark(&tombstone), RetirementMark::Absent); - - let store = open(&dir).await; - assert!( - store.has_legacy(), - "an unmarked environment must be restored and served, not deleted" - ); - assert!(env.exists(), "it must be back under its own name"); - for key in &keys { - assert!(store.get(key).await.expect("get").is_some()); - } - } + use tempfile::TempDir; - /// A mark already at that name is not a mark until it can be read. - /// - /// The one place in this file that was taking the name as the evidence, which is the - /// thing every other part of it refuses to do. Retirement writes the mark with - /// `create_new`, and a failure saying something is already there was accepted as "the - /// mark is present" and the environment deleted on the strength of it. What is at that - /// name might be anything. - /// - /// It matters most for a node that already has its store open. That path never - /// consulted the mark at all until this round, so an unreadable one would have gone - /// through every gate and been deleted. - #[cfg(unix)] - #[test] - fn a_mark_already_at_that_name_is_not_accepted_until_it_can_be_read() { + /// Open a store on a fresh temp directory with the disk reserve disabled. + async fn test_store() -> (ChunkStore, TempDir) { let dir = TempDir::new().expect("temp dir"); - let env = dir.path().join(LEGACY_ENV_DIR); - std::fs::create_dir_all(&env).expect("mkdir"); - make_the_mark_unreadable(&env); - - let refused = mark_directory_retired(&env).expect_err("an unreadable mark is not a mark"); - assert!( - !refused.mark_definitely_gone, - "something is at that name, so the caller must not treat it as absent and put \ - the directory back under a name that will be opened" - ); - assert_eq!(retirement_mark(&env), RetirementMark::Unknown); + let store = ChunkStore::new(ChunkStoreConfig { + root_dir: dir.path().to_path_buf(), + verify_on_read: true, + disk_reserve: 0, + }) + .await + .expect("open store"); + (store, dir) + } - // And a real one is still accepted, so the refusal above is about being unable to - // read it rather than about there being something there at all. - std::fs::remove_file(env.join(RETIRED_MARKER)).expect("clear the link"); - mark_directory_retired(&env).expect("a first mark"); - mark_directory_retired(&env).expect("and the same mark again, which is readable"); - assert_eq!(retirement_mark(&env), RetirementMark::Present); + /// Open a store on an existing directory, as a restart would. + async fn reopen(dir: &TempDir) -> ChunkStore { + ChunkStore::new(ChunkStoreConfig { + root_dir: dir.path().to_path_buf(), + verify_on_read: true, + disk_reserve: 0, + }) + .await + .expect("reopen store") } - /// A directory nothing can classify is neither restored nor deleted. + /// An ordinary read settles whether the node answers for a chunk. /// - /// The half of the three-state answer that a first attempt at this got wrong. Asking - /// only "may it be removed" and letting everything else fall through puts "cannot tell" - /// straight back on the restoring path, which is the resurrection this exists to - /// prevent: the mark check can fail for a moment and succeed the next, and the restore - /// in between brings back a store that really had been retired. - /// - /// Unix only: the state is staged with a symbolic link, which Windows does not offer - /// on the same terms. + /// Not only the reads that were checking something. A read that failed means the + /// chunk cannot be served, whoever asked; a read that worked means it can be. Deciding + /// this anywhere else leaves a key stuck unadvertised after the fault has cleared, or + /// advertised after it has not. #[cfg(unix)] #[tokio::test] - async fn a_tombstone_that_cannot_be_classified_is_left_where_it_is() { - let dir = TempDir::new().expect("temp dir"); - seed_legacy(&dir, &["unclassifiable"]).await; - let env = dir.path().join(LEGACY_ENV_DIR); - let tombstone = dir.path().join(format!("{LEGACY_ENV_DIR}{RETIRED_SUFFIX}")); - std::fs::rename(&env, &tombstone).expect("rename"); - make_the_mark_unreadable(&tombstone); - assert_eq!(retirement_mark(&tombstone), RetirementMark::Unknown); + async fn an_ordinary_read_decides_whether_the_node_answers_for_a_chunk() { + use std::os::unix::fs::PermissionsExt; + + let (store, dir) = test_store().await; + let (addr, content) = addressed("read-decides"); + store.put(&addr, &content).await.expect("put"); + let path = store.chunk_path(&addr); - sweep_retired_legacy(dir.path()); - let still_there = tombstone.exists(); - let restored = env.exists(); + let mut perms = std::fs::metadata(&path).expect("meta").permissions(); + perms.set_mode(0o000); + std::fs::set_permissions(&path, perms).expect("chmod"); + assert!(store.get(&addr).await.is_err(), "the read must fail"); assert!( - still_there, - "a directory that cannot be classified must not be deleted: it may be intact" - ); - assert!( - !restored, - "a directory that cannot be classified must not be put back under the live \ - name: it may be half deleted" - ); - } - - /// The same directory under the live name is not opened either. - /// - /// Opening it would put keys back into a commitment they may already have left, and - /// this node cannot tell whether they have. Serving from files alone is the answer that - /// is right either way. - #[cfg(unix)] - #[test] - fn a_live_environment_that_cannot_be_classified_is_not_opened() { - let dir = TempDir::new().expect("temp dir"); - let env = dir.path().join(LEGACY_ENV_DIR); - std::fs::create_dir_all(&env).expect("mkdir"); - std::fs::write(env.join("data.mdb"), b"not really an environment").expect("seed"); - assert_eq!( - finish_interrupted_retirement(dir.path()), - LiveEnvironment::WhateverIsOnDisk, - "a readable directory with no mark is ordinary and may be opened" + !store.exists(&addr).expect("exists"), + "and a plain read that failed must stop the node answering for it" ); - make_the_mark_unreadable(&env); - assert_eq!(retirement_mark(&env), RetirementMark::Unknown); - - let verdict = finish_interrupted_retirement(dir.path()); - let still_there = env.exists(); + let mut perms = std::fs::metadata(&path).expect("meta").permissions(); + perms.set_mode(0o600); + std::fs::set_permissions(&path, perms).expect("chmod back"); assert_eq!( - verdict, - LiveEnvironment::None, - "a directory that cannot be classified must not be opened" + store.get(&addr).await.expect("get").expect("present"), + content ); assert!( - still_there, - "and it must not be deleted either: it may be a live environment" + store.exists(&addr).expect("exists"), + "and a plain read that worked must start it answering again" ); + drop(dir); } - /// Make the retirement mark in `dir` unreadable without touching the directory itself. - /// - /// A symbolic link pointing at itself. Looking for the mark follows it, gets - /// `FilesystemLoop` back, and the answer is neither "there" nor "not there", which is - /// the state under test. + /// Two writes for one key: waiting means waiting for both. /// - /// Taking the directory's permissions away instead was the first attempt and staged too - /// much: at mode 000 the operating system refuses the rename as well, so the code being - /// tested was never reached and the test passed with its own protection removed. Root - /// can also read a mode-000 directory, which would have made it fail on any CI that - /// runs as root. A link loop is neither: everything else about the directory keeps - /// working, for every user. - #[cfg(unix)] - fn make_the_mark_unreadable(dir: &Path) { - let link = dir.join(RETIRED_MARKER); - let _ = std::fs::remove_file(&link); - std::os::unix::fs::symlink(&link, &link).expect("a link to itself"); - } - - /// Both names taken is not a decision this code makes. + /// Cancellation releases the caller's lane while the blocking half survives, so a + /// second write for the same key can start behind the first. If the registry only + /// recorded that *something* was writing, whichever finished first would clear it and + /// a delete would be told the key was free while the other was still queued, then be + /// undone by it. #[tokio::test] - async fn an_unmarked_retired_directory_beside_a_live_one_is_left_alone() { - let dir = TempDir::new().expect("temp dir"); - seed_legacy(&dir, &["live"]).await; - let tombstone = dir.path().join(format!("{LEGACY_ENV_DIR}{RETIRED_SUFFIX}")); - std::fs::create_dir_all(&tombstone).expect("mkdir"); - std::fs::write(tombstone.join("data.mdb"), b"something").expect("write"); - - let store = open(&dir).await; - assert!(store.has_legacy()); - assert!( - tombstone.exists(), - "an unmarked directory must never be deleted, even beside a live one" - ); - assert!(dir.path().join(LEGACY_ENV_DIR).exists()); - } - - /// The mark is the last thing a deletion takes away. - /// - /// A recursive delete walks in whatever order the filesystem gives, so it can unlink - /// the mark and then fail on the next entry, which is what a sharing violation on the - /// data file looks like. That leaves a genuinely retired, partly deleted directory - /// carrying no evidence of it, and the next start would read that as intact and - /// restore it. - /// Unix only: the failure is provoked with directory permissions, which is not how - /// the same thing happens on Windows. The behaviour under test is platform-neutral. - #[cfg(unix)] - #[test] - fn a_failed_deletion_leaves_the_mark_in_place() { - use std::os::unix::fs::PermissionsExt; - - let dir = TempDir::new().expect("temp dir"); - let retired = dir.path().join("chunks.mdb.retired"); - std::fs::create_dir_all(&retired).expect("mkdir"); - std::fs::write(retired.join(LEGACY_DATA_FILE), b"payload").expect("write"); - mark_directory_retired(&retired).expect("mark"); - - // An entry that cannot be removed, standing in for whatever the filesystem - // refuses on the day. - std::fs::create_dir_all(retired.join("stuck")).expect("mkdir"); - let mut perms = std::fs::metadata(&retired).expect("meta").permissions(); - perms.set_mode(0o500); - std::fs::set_permissions(&retired, perms.clone()).expect("chmod"); + async fn waiting_for_a_key_waits_for_every_write_of_it() { + let (store, dir) = test_store().await; + let store = Arc::new(store); + let (addr, content) = addressed("two-writers"); - let failed = remove_marked_directory(&retired).is_err(); + // Two registrations, as two overlapping writes would make. + let first = store.begin_write(&addr); + let second = store.begin_write(&addr); - perms.set_mode(0o700); - std::fs::set_permissions(&retired, perms).expect("chmod back"); + let waiting = { + let store = Arc::clone(&store); + tokio::spawn(async move { store.wait_for_write(&addr).await }) + }; + tokio::time::sleep(Duration::from_millis(50)).await; + assert!(!waiting.is_finished()); - assert!(failed, "the deletion was supposed to fail"); + // One finishes. The other has not, so the wait must continue. + drop(first); + tokio::time::sleep(Duration::from_millis(50)).await; assert!( - retirement_mark(&retired) == RetirementMark::Present, - "a deletion that failed must leave the mark, or the directory stops saying \ - what it is" + !waiting.is_finished(), + "one write finishing does not mean the key is free" ); + + drop(second); + waiting + .await + .expect("the wait ends once both have finished"); + + // And the store is still usable afterwards. + store.put(&addr, &content).await.expect("put"); + assert!(store.exists(&addr).expect("exists")); + drop(dir); } - /// A mark that cannot be read is not permission to do anything. - /// - /// The reason this is three states and not two. Reading the mark can fail for reasons - /// that are neither yes nor no, and the old answer for those was "no mark", which is - /// the worst of the three: a retired environment reads as live, goes back under its own - /// name, and its keys re-enter a commitment they have already left. Deleting on an - /// unreadable answer would be just as wrong in the other direction. + /// A chunk this store cannot read is kept but not claimed. /// - /// Unix only: the state is staged with a symbolic link, which Windows does not offer - /// on the same terms. + /// Both halves matter. Deleting it, or dropping it from the index, is how a chunk ends + /// up in neither this store's view nor the legacy one, which is what retirement + /// destroys. Claiming it anyway puts the key in signed commitments and answers + /// presence probes with a yes for a chunk the node cannot serve, and the audit that + /// catches that still penalises. #[cfg(unix)] - #[test] - fn a_mark_that_cannot_be_read_permits_nothing() { - let dir = TempDir::new().expect("temp dir"); - let env = dir.path().join(LEGACY_ENV_DIR); - - // Nothing there is not the same as cannot tell, and it is the ordinary case: every - // node that never had a legacy store, and every node that has finished with one, - // asks this question on every tick. Answering "cannot tell" for those would have a - // fresh node run a migration driver forever over a store it does not have. - assert_eq!( - retirement_mark(&env), - RetirementMark::Absent, - "a directory that is not there carries no mark, and that is known" - ); - assert!(retirement_mark(&env).permits_opening()); + #[tokio::test] + async fn a_chunk_that_cannot_be_read_is_kept_but_not_claimed() { + use std::os::unix::fs::PermissionsExt; - std::fs::create_dir_all(&env).expect("mkdir"); - std::fs::write(env.join(RETIRED_MARKER), b"retired").expect("mark"); - assert_eq!(retirement_mark(&env), RetirementMark::Present); + let (store, dir) = test_store().await; + let (addr, content) = addressed("unreadable-for-now"); + store.put(&addr, &content).await.expect("put"); + assert!(store.exists(&addr).expect("exists")); - // Looking for the mark now goes round in a circle, so the answer is neither there - // nor not there. - make_the_mark_unreadable(&env); - let unreadable = retirement_mark(&env); + let path = store.chunk_path(&addr); + let mut perms = std::fs::metadata(&path).expect("meta").permissions(); + perms.set_mode(0o000); + std::fs::set_permissions(&path, perms).expect("chmod"); - assert_eq!( - unreadable, - RetirementMark::Unknown, - "a mark that cannot be read must not report as absent" - ); + // Offering the same bytes again must not be acknowledged, and must not replace + // what is there on the strength of a read that did not happen. assert!( - !unreadable.permits_removal(), - "an unreadable mark must not authorise deleting the environment" + store.put(&addr, &content).await.is_err(), + "an unreadable chunk must not be reported as stored" ); + assert!(path.exists(), "and the file must be left alone"); assert!( - !unreadable.permits_opening(), - "an unreadable mark must not authorise reopening the environment" + !store.exists(&addr).expect("exists"), + "but the node must stop claiming it" ); - } + assert!(!store.all_keys().await.expect("keys").contains(&addr)); - /// A directory under the live name that says it was retired is never opened. - /// - /// It may be partly deleted, and opening it would put keys back into a commitment - /// they have already left. Its chunks are in the file store, which is what the mark - /// records, so serving from files alone is correct. - #[tokio::test] - async fn a_marked_directory_under_the_live_name_is_not_served_from() { - let dir = TempDir::new().expect("temp dir"); - let keys = seed_legacy(&dir, &["marked-live"]).await; - { - let store = open(&dir).await; - store - .copy_batch(&keys, 0, 0, &never_cancelled()) - .await - .expect("copy"); - } - let env = dir.path().join(LEGACY_ENV_DIR); - mark_directory_retired(&env).expect("mark"); - - // Every name it could be moved to is taken by something that is not empty, so the - // rename fails and the marked directory stays under the live name. That is the - // case this is about: it must be left alone rather than opened. - for n in 0..=MAX_TOMBSTONES { - let taken = if n == 0 { - dir.path().join(format!("{LEGACY_ENV_DIR}{RETIRED_SUFFIX}")) - } else { - dir.path() - .join(format!("{LEGACY_ENV_DIR}{RETIRED_SUFFIX}.{n}")) - }; - std::fs::create_dir_all(&taken).expect("mkdir"); - std::fs::write(taken.join("occupied"), b"x").expect("write"); - } + // Readable again: the node answers for it once more. + let mut perms = std::fs::metadata(&path).expect("meta").permissions(); + perms.set_mode(0o600); + std::fs::set_permissions(&path, perms).expect("chmod back"); + assert!(!store.put(&addr, &content).await.expect("put again")); + assert!(store.exists(&addr).expect("exists")); + assert!(store.all_keys().await.expect("keys").contains(&addr)); + drop(dir); + } - let store = open(&dir).await; - assert!( - env.exists(), - "the rename was supposed to fail, leaving the marked directory in place" - ); - assert!( - !store.has_legacy(), - "a directory that says it was retired must never be opened as live" - ); - for key in &keys { - assert!(store.get(key).await.expect("get").is_some()); - } + /// Content plus the address it hashes to. + fn addressed(seed: &str) -> (XorName, Vec) { + let content = format!("chunk-content-{seed}").into_bytes(); + (crate::client::compute_address(&content), content) } - /// A linked environment is copied out of but never deleted. + /// A value over the ceiling is not a chunk, and this build will not hold one anywhere. /// - /// An operator who points the chunk store at another volume leaves a link here. - /// Retirement renames the path and then deletes what is behind it, and behind a link - /// is a directory somewhere else that this node does not own. - #[cfg(unix)] + /// The 4 MB ceiling is what makes a chunk a chunk, and every released ingress enforces it + /// before anything is stored: the protocol handler on a paid store, and replication on + /// both the receive and the fetch path. So no over-ceiling value ever entered this network + /// as a chunk, and one found on a disk is not data to be preserved — it is not a chunk at + /// all. + /// + /// There was exactly one way to get one onto a disk, and it was ours, not the network's. + /// The bridge's `put` wrote to the legacy environment FIRST — LMDB has no size ceiling — + /// and only then offered the same bytes to the file store, which refused them. It then + /// recorded the key as legacy-only so the copier would retry it, and the copier's own + /// size arm deleted it. A local caller of the public `ChunkStore::put` was the whole of + /// the exposure. + /// + /// That path is gone with the bridge. There is one store, its `put` refuses over the + /// ceiling, and it refuses BEFORE it writes anything, so there is no half-written state + /// to reason about and nothing for a later pass to find and have to decide about. This is + /// why the release preserves no such value and grows no sidecar to put one in: there is + /// no valid chunk to lose, and building a place to keep invalid ones would be building + /// for a case this release makes unreachable. #[tokio::test] - async fn a_linked_environment_is_never_retired() { - let outside = TempDir::new().expect("temp dir"); - let dir = TempDir::new().expect("temp dir"); - // A real environment that lives in `outside`; the node root only links to it. - seed_legacy(&outside, &["someone-elses"]).await; - let real = outside.path().join(LEGACY_ENV_DIR); - let bystander = outside.path().join("unrelated"); - std::fs::create_dir_all(&bystander).expect("mkdir"); - std::os::unix::fs::symlink(&real, dir.path().join(LEGACY_ENV_DIR)).expect("symlink"); - - let store = open(&dir).await; - let blocker = store - .retirement_blocker(|_| false) - .expect("a linked environment must block retirement"); + async fn a_value_over_the_ceiling_is_refused_before_anything_is_written() { + let (store, dir) = test_store().await; + + // Addressed to its own bytes, so the content-address arm passes and the refusal is + // the size one. Checking the wrong arm would pass this test while the ceiling was + // gone. + let content = vec![0xAB; MAX_CHUNK_SIZE + 1]; + let address = crate::client::compute_address(&content); + + let err = store + .put(&address, &content) + .await + .expect_err("a value over the ceiling is not a chunk and must be refused"); + let message = format!("{err}"); assert!( - blocker.contains("link"), - "the reason must name the actual problem, got: {blocker}" + message.contains("byte maximum"), + "refused for the wrong reason, so this proves nothing about the ceiling: {message}" ); - // And nothing walks through it, whatever it is marked with. - std::fs::write(real.join(RETIRED_MARKER), b"x").expect("mark through the link"); assert!( - retirement_mark(&dir.path().join(LEGACY_ENV_DIR)) != RetirementMark::Present, - "a link must never be treated as a retired directory" + !store.exists(&address).expect("exists"), + "the node must not claim a value it refused" ); + assert!(!store.all_keys().await.expect("keys").contains(&address)); assert!( - remove_marked_directory(&dir.path().join(LEGACY_ENV_DIR)).is_err(), - "deleting through a link must be refused" + !store.chunk_path(&address).exists(), + "nothing may be left on disk: the refusal comes before the write" ); - assert!( - real.join(LEGACY_DATA_FILE).exists(), - "and must delete nothing" + + // And a restart cannot find one either, which is what says the refusal left no + // partial file for a startup scan to index by name. The store holds its directory + // lock for as long as it lives, so it goes first: two of them on one root is refused, + // which is a different failure and would prove nothing about the ceiling. + drop(store); + let reopened = reopen(&dir).await; + assert!(!reopened.exists(&address).expect("exists")); + assert_eq!( + reopened.current_chunks().expect("count"), + 0, + "a refused value must not survive as a name the scan believes" ); - assert!(bystander.exists()); } - /// A deletion that cannot remove the directory itself puts the mark back. - /// - /// An unmarked directory that still exists is the one state the scheme says cannot - /// happen: the next start would read it as an intact environment. - /// - /// Unix only: the failure is provoked with directory permissions, which is not how - /// the same thing happens on Windows. The behaviour under test is platform-neutral. - #[cfg(unix)] - #[test] - fn a_directory_that_cannot_be_removed_keeps_saying_it_was_retired() { - use std::os::unix::fs::PermissionsExt; + #[tokio::test] + async fn put_then_get_returns_the_same_bytes() { + let (store, _dir) = test_store().await; + let (addr, content) = addressed("a"); - let dir = TempDir::new().expect("temp dir"); - let retired = dir.path().join("chunks.mdb.retired"); - std::fs::create_dir_all(&retired).expect("mkdir"); - std::fs::write(retired.join(LEGACY_DATA_FILE), b"payload").expect("write"); - mark_directory_retired(&retired).expect("mark"); - - // The parent read-only, so the directory cannot be unlinked from it while its own - // contents still can be. That is the shape that leaves an emptied, unmarked - // directory behind. - let mut perms = std::fs::metadata(dir.path()).expect("meta").permissions(); - perms.set_mode(0o500); - std::fs::set_permissions(dir.path(), perms).expect("chmod"); - - let failed = remove_marked_directory(&retired).is_err(); - - let mut perms = std::fs::metadata(dir.path()).expect("meta").permissions(); - perms.set_mode(0o700); - std::fs::set_permissions(dir.path(), perms).expect("chmod back"); - - assert!(failed, "the removal was supposed to fail"); - assert!(retired.exists(), "and to leave the directory behind"); - assert!( - retirement_mark(&retired) == RetirementMark::Present, - "a directory that outlived its deletion must still say what it is" - ); + assert!(store.put(&addr, &content).await.expect("put")); + let got = store.get(&addr).await.expect("get").expect("present"); + assert_eq!(got, content); } - /// A key the environment holds that is in neither view stops retirement. - /// - /// The gates only ever see the legacy-only set, so a key that has fallen out of both - /// the file index and that set has been through nothing and is protected by nothing. - /// It is the environment's only copy, and retirement would take it. #[tokio::test] - async fn a_legacy_key_in_neither_view_refuses_the_proof_and_is_re_queued() { - let dir = TempDir::new().expect("temp dir"); - let keys = seed_legacy(&dir, &["orphan"]).await; - let key = *keys.first().expect("one key"); - let store = open(&dir).await; - store - .copy_batch(&keys, 0, 0, &never_cancelled()) - .await - .expect("copy"); - assert!(!store.legacy_only_keys().contains(&key)); + async fn a_second_put_of_the_same_chunk_reports_not_new() { + let (store, _dir) = test_store().await; + let (addr, content) = addressed("b"); - // Stand in for whatever takes the file out from under the index: a quarantine, a - // publish that failed, an operator. The key is now in neither view. - let path = dir - .path() - .join("chunks") - .join(format!("{:02x}", key.last().copied().unwrap_or(0))) - .join(hex::encode(key)); - std::fs::remove_file(&path).expect("remove the file"); - store.files.forget_for_test(&key); - assert!(!store.files.exists(&key).unwrap_or(false)); - assert!(!store.legacy_only_keys().contains(&key)); - - let proof = store - .verify_before_retire(0, &never_cancelled()) - .await - .expect("verify"); - assert!( - !proof.is_clean(), - "a key protected by neither view must refuse the proof" - ); - assert!( - store.legacy_only_keys().contains(&key), - "and must be put back where the gates can see it" - ); + assert!(store.put(&addr, &content).await.expect("first put")); + assert!(!store.put(&addr, &content).await.expect("second put")); + assert_eq!(store.current_chunks().expect("count"), 1); + assert_eq!(store.stats().duplicates, 1); } - /// A verification that no longer describes the store does not authorise a deletion. - /// - /// The pass reads every chunk and its result is reused for a while rather than re-read - /// on every tick. Retirement is often deferred in that window by a gate that has - /// nothing to do with the files. If a kept chunk stops being readable meanwhile, - /// ordinary requests are still served from the legacy copy, and deleting that copy on - /// the strength of the older pass leaves the node holding only the unreadable one. - #[cfg(unix)] #[tokio::test] - async fn a_verification_overtaken_by_a_failing_file_does_not_authorise_retirement() { - use std::os::unix::fs::PermissionsExt; + async fn get_of_an_unknown_address_is_none() { + let (store, _dir) = test_store().await; + let (addr, _) = addressed("missing"); + assert!(store.get(&addr).await.expect("get").is_none()); + } - let dir = TempDir::new().expect("temp dir"); - let keys = seed_legacy(&dir, &["kept-then-unreadable"]).await; - let key = *keys.first().expect("one key"); - let mut config = test_config(&dir); - config.migration.retire_legacy = true; - let store = ChunkStore::new(config).await.expect("open"); - store - .copy_batch(&keys, 0, 0, &never_cancelled()) - .await - .expect("copy"); - open_the_retirement_gate(&store); + #[tokio::test] + async fn exists_tracks_the_store() { + let (store, _dir) = test_store().await; + let (addr, content) = addressed("c"); - let proof = store - .verify_before_retire(0, &never_cancelled()) - .await - .expect("verify"); - assert!(proof.is_clean()); + assert!(!store.exists(&addr).expect("exists")); + store.put(&addr, &content).await.expect("put"); + assert!(store.exists(&addr).expect("exists")); + store.delete(&addr).await.expect("delete"); + assert!(!store.exists(&addr).expect("exists")); + } - // The window: the file stops being readable after the pass and before the - // deletion it authorised. - let path = dir - .path() - .join("chunks") - .join(format!("{:02x}", key.last().copied().unwrap_or(0))) - .join(hex::encode(key)); - let mut perms = std::fs::metadata(&path).expect("meta").permissions(); - perms.set_mode(0o000); - std::fs::set_permissions(&path, perms).expect("chmod"); - assert!( - store.get(&key).await.is_ok(), - "the legacy copy still serves it, which is what hides the problem" - ); + #[tokio::test] + async fn delete_unlinks_the_file_and_returns_the_space() { + let (store, _dir) = test_store().await; + let (addr, content) = addressed("d"); + store.put(&addr, &content).await.expect("put"); - let err = store - .retire_legacy(&proof, &|_: &XorName| false, &approved_shed(&store)) - .await - .expect_err("a verification the store has outrun must not authorise a delete"); - assert!(format!("{err}").contains("no longer describes"), "{err}"); - assert!( - store.has_legacy(), - "and the legacy environment must survive" - ); + let path = store.chunk_path(&addr); + assert!(path.exists(), "the chunk file should be on disk"); - let mut perms = std::fs::metadata(&path).expect("meta").permissions(); - perms.set_mode(0o600); - std::fs::set_permissions(&path, perms).expect("chmod back"); + assert!(store.delete(&addr).await.expect("delete")); + assert!(!path.exists(), "delete must actually unlink the file"); + assert_eq!(store.current_chunks().expect("count"), 0); + + // Deleting again is a no-op that reports nothing was there. + assert!(!store.delete(&addr).await.expect("second delete")); } - /// A write with no rollback copy is counted, on the path that actually skips. - /// - /// The environment is pinned to its current size for the whole bridge, so one with no - /// reusable page answers `Full` to every write and the node stores in files alone. That - /// is accepted, and the ADR says so. What was missing was any way to ask how often it - /// happens: the second release turns on knowing how many nodes are really keeping a - /// rollback copy, and a log line per chunk does not answer it. - /// - /// The first version of this counter missed exactly this path and counted only the - /// other one, the write that is attempted and refused. On the node most affected the - /// other path never runs, so the counter stayed at zero on precisely the nodes it was - /// added for. #[tokio::test] - async fn a_write_with_no_rollback_copy_is_counted() { - let dir = TempDir::new().expect("temp dir"); - seed_legacy(&dir, &["settled"]).await; - let store = open(&dir).await; - assert_eq!( - store.writes_without_a_rollback_copy(), - 0, - "nothing has been skipped yet" - ); - - let legacy = store.legacy().expect("legacy"); - let before = legacy - .skipped_rollback_copies - .load(std::sync::atomic::Ordering::Relaxed); - - // Whichever way the environment refuses, the count moves. Driven through the - // counter itself rather than by filling a real environment, because what is under - // test is that the skip is recorded, and staging a genuinely unwritable LMDB from - // here would be testing LMDB. - for _ in 0..12 { - let _ = legacy.note_skipped_rollback_copy(); - } - assert_eq!( - store.writes_without_a_rollback_copy(), - before + 12, - "skipped writes must be visible to whoever asks the node" - ); - - // And the log is throttled, or a node with no free pages writes one line per chunk - // for the rest of its life. - let said: Vec = (0..1_000) - .filter_map(|_| legacy.note_skipped_rollback_copy()) - .collect(); + async fn content_that_does_not_hash_to_its_address_is_rejected() { + let (store, _dir) = test_store().await; + let (addr, _) = addressed("e"); + let err = store + .put(&addr, b"different content") + .await + .expect_err("must reject"); assert!( - said.len() < 10, - "the warning fired {} times in a thousand writes", - said.len() + format!("{err}").contains("Content address mismatch"), + "unexpected error: {err}" ); + assert_eq!(store.current_chunks().expect("count"), 0); } - /// Two writes for one key need two notes, not one shared between them. - /// - /// The journal used to be a set, so a second write for the same key announced nothing - /// and the first to return cleared the entry for both. A delete arriving in that window - /// sees no announcement, skips draining the environment, and the surviving write lands - /// afterwards and puts the key back, undoing a prune the node had decided on. The key - /// then sits in the environment and in neither view, which is the state retirement is - /// built to refuse: no data is lost, but a prune and a retirement cycle are. - /// - /// The file store's own in-flight map is counted for exactly this reason. This is the - /// same reasoning applied to the half that did not have it. #[tokio::test] - async fn two_writes_for_one_key_are_two_notes_and_the_first_to_return_clears_neither() { - let dir = TempDir::new().expect("temp dir"); - seed_legacy(&dir, &["settled"]).await; - let store = open(&dir).await; - let legacy = store.legacy().expect("legacy"); - let (addr, _) = addressed("two-writes"); - - legacy.announce(&addr); - legacy.announce(&addr); - assert!(store.has_pending_writes()); + async fn a_chunk_is_filed_under_the_last_two_hex_characters_of_its_address() { + let (store, dir) = test_store().await; + let (addr, content) = addressed("f"); + store.put(&addr, &content).await.expect("put"); - // The first write returns. The second is still out there, so the note has to stand. - legacy.announced_write_finished(&addr); - assert!( - store.has_pending_writes(), - "the first write to return cleared a note the second one was still relying on" - ); + let name = hex::encode(addr); + let expected_shard = name + .get(name.len() - 2..) + .expect("64-character name") + .to_string(); + let path = dir + .path() + .join(CHUNKS_DIR_NAME) + .join(&expected_shard) + .join(&name); + assert!(path.exists(), "expected the chunk at {}", path.display()); + } - // And the second clears it. - legacy.announced_write_finished(&addr); - assert!(!store.has_pending_writes()); + #[tokio::test] + async fn the_index_is_rebuilt_from_the_filesystem_on_restart() { + let (store, dir) = test_store().await; + let mut written = Vec::new(); + for i in 0..64 { + let (addr, content) = addressed(&format!("restart-{i}")); + store.put(&addr, &content).await.expect("put"); + written.push(addr); + } + drop(store); - // Retiring one that was never announced changes nothing, which is what makes the - // delete path's unconditional clear safe. - legacy.announced_write_finished(&addr); - assert!(!store.has_pending_writes()); + let reopened = reopen(&dir).await; + assert_eq!(reopened.current_chunks().expect("count"), 64); + for addr in &written { + assert!(reopened.exists(addr).expect("exists"), "lost a key"); + } } - /// A write in flight is a note to self, not a claim to hold the chunk. - /// - /// The note exists because a write into the environment outlives the future waiting - /// for it, so a cancelled one could leave a chunk nothing had recorded. But until both - /// halves have returned the node does not hold it, and saying it does puts the key in - /// signed commitments and in the count a quote is priced from. #[tokio::test] - async fn a_write_in_flight_is_not_claimed_but_does_stop_retirement() { - let dir = TempDir::new().expect("temp dir"); - seed_legacy(&dir, &["settled"]).await; - let store = open(&dir).await; - let legacy = store.legacy().expect("legacy"); - let (addr, _) = addressed("in-flight"); + async fn all_keys_is_sorted_ascending() { + let (store, dir) = test_store().await; + for i in 0..128 { + let (addr, content) = addressed(&format!("sorted-{i}")); + store.put(&addr, &content).await.expect("put"); + } - // Stand in for a write that announced itself and never came back. - legacy.announce(&addr); + let keys = store.all_keys().await.expect("all_keys"); + let mut sorted = keys.clone(); + sorted.sort_unstable(); + assert_eq!(keys, sorted, "all_keys() must be ordered"); - assert!( - !store.exists(&addr).expect("exists"), - "a write in flight must not be reported as held" - ); - assert!(!store.all_keys().await.expect("keys").contains(&addr)); - assert!(!store.legacy_only_keys().contains(&addr)); - assert!(store.has_pending_writes()); + // And the order has to survive a restart, because the commitment builder + // truncates the responsible subset before the Merkle tree sorts it. + drop(store); + let reopened = reopen(&dir).await; + assert_eq!(reopened.all_keys().await.expect("all_keys"), keys); + } - // The distinction is the point: the same key in the key set IS claimed. If a - // write announced itself there instead, every one of the assertions above would - // be the opposite for as long as the write took. - legacy.only.write().insert(addr); - assert!(store.exists(&addr).expect("exists")); - assert!(store.all_keys().await.expect("keys").contains(&addr)); - legacy.only.write().remove(&addr); - - // But it does stop the environment going, because what it holds is unsettled. - store.commit_to_files().expect("commit"); - store.note_commitment_rebuilt(); - store.note_commitment_rebuilt(); - store.force_migration_state(|s| { - s.committed_at_unix = - Some(now_unix().saturating_sub(MIN_RETIRE_DELAY_HOURS * 3600 + 60)); - }); - let proof = store - .verify_before_retire(0, &never_cancelled()) - .await - .expect("verify"); - let err = store - .retire_legacy(&proof, &|_: &XorName| false, &approved_shed(&store)) - .await - .expect_err("an unsettled write must stop the removal"); - assert!(format!("{err}").contains("not reported back"), "{err}"); + #[tokio::test] + async fn get_raw_skips_verification() { + let (store, _dir) = test_store().await; + let (addr, content) = addressed("raw"); + store.put(&addr, &content).await.expect("put"); - // And the note is resolved against what is actually there: nothing, so it goes. - store.reconcile_pending_writes().await; - assert!(!store.has_pending_writes()); - assert!(!store.legacy_only_keys().contains(&addr)); + // Corrupt the file behind the store's back. + std::fs::write(store.chunk_path(&addr), b"tampered").expect("tamper"); + + let raw = store.get_raw(&addr).await.expect("get_raw").expect("bytes"); + assert_eq!(raw, b"tampered"); } - /// A delete outlasts a write for the same key that nobody waited for. + /// A delete waits for a put that is already under way for the same key. + /// + /// The narrower ordering, and the one waiting for registered writes does not cover. A + /// put does a lot before it registers itself: it checks the address, reads to see + /// whether the name is taken, and reserves capacity. A delete arriving in that window + /// would see nothing registered, wait for nothing, and go ahead; the put would register + /// and publish afterwards, and the node would keep a chunk it had decided to prune. /// - /// A write has an environment half and a file half, and either can still be running - /// when its caller is dropped: the blocking work is not cancelled with the future. - /// A delete that did not wait for both would be undone by whichever half landed - /// afterwards, putting back a chunk the node had decided to prune. + /// Staged deterministically rather than by racing two tasks. A put parked inside its + /// own closure is holding the key's lane, so the delete must not be able to finish + /// while it is parked. An earlier version of this test started both and accepted either + /// ordering, which the bug also satisfies: it proved nothing. #[tokio::test] - async fn a_delete_outlasts_a_write_nobody_waited_for() { + #[allow(clippy::await_holding_lock)] + async fn a_delete_waits_for_a_put_already_under_way() { let dir = TempDir::new().expect("temp dir"); - seed_legacy(&dir, &["neighbour"]).await; - let store = Arc::new(open(&dir).await); - let (addr, content) = addressed("written-then-pruned"); - - // The gate is held from its own thread, so nothing holds a blocking guard across - // an await, and it is released through a channel when the test is ready. - let gate = store.files.test_put_gate(); - let (held_tx, held_rx) = std::sync::mpsc::channel::<()>(); - let (release_tx, release_rx) = std::sync::mpsc::channel::<()>(); - let holder = std::thread::spawn(move || { - let _parked = gate.write(); - held_tx.send(()).ok(); - release_rx.recv().ok(); - }); - held_rx.recv().expect("the gate is held"); - - // The state under test, built directly rather than by racing a real put: a write - // that announced itself, whose file half is parked mid-publish, and whose caller - // is gone. Driving it through `put` and aborting would be a race about which half - // had started, and a test that sometimes sets up a different state than it claims - // is worse than no test. - let legacy = store.legacy().expect("legacy"); - legacy.announce(&addr); - let publishing = { - let files = Arc::clone(&store.files); + let store = Arc::new(reopen(&dir).await); + let content = b"a chunk the pruner has decided to drop".to_vec(); + let addr = crate::client::compute_address(&content); + + // Park the put in the window that matters: it has taken the key's lane and has NOT + // yet registered itself, so a delete's wait for in-flight writes would see nothing. + // Parking it later, inside its closure, cannot show the lane doing anything: the + // delete would block on the wait instead and the test would pass either way. + let gate = store.test_pre_registration_gate(); + let held = gate.write().await; + let writing = { + let store = Arc::clone(&store); let content = content.clone(); - tokio::spawn(async move { files.put(&addr, &content).await }) + tokio::spawn(async move { store.put(&addr, &content).await }) }; - // Until the publish is genuinely in flight and parked at the gate. - while store.files.tasks_in_flight() == 0 { + // Waited for, not slept at. A sleep makes the staging a guess, and on a loaded + // machine the guess is wrong and the test fails for the wrong reason. + let deadline = std::time::Instant::now() + Duration::from_secs(30); + while store.test_reached_pre_registration() == 0 { + assert!( + std::time::Instant::now() < deadline, + "the put never reached the gate" + ); tokio::time::sleep(Duration::from_millis(5)).await; } - publishing.abort(); - let _ = publishing.await; - // The delete has to wait the parked half out rather than racing it. + // The delete must not get past the lane while that put holds it. Without the lane + // on `put` this finishes immediately, which is the regression. let deleting = { let store = Arc::clone(&store); tokio::spawn(async move { store.delete(&addr).await }) }; - tokio::time::sleep(Duration::from_millis(100)).await; + tokio::time::sleep(Duration::from_millis(200)).await; assert!( !deleting.is_finished(), - "the delete must wait for the write it would otherwise race" + "the delete finished while a put for the same key was still under way" ); - release_tx.send(()).ok(); - holder.join().ok(); - deleting.await.expect("join").expect("delete"); + // Released: the put completes, then the delete runs. The delete is the later + // decision, so the chunk must be gone. + drop(held); + let _ = writing.await.expect("the put task must not panic"); + let _ = deleting.await.expect("the delete task must not panic"); + store.wait_idle().await; - // Whichever half landed, the key is gone and stays gone. - store.files.wait_idle().await; assert!( - !store.exists(&addr).unwrap_or(true), - "a write that landed after the delete would resurrect a pruned chunk" + !store.is_indexed(&addr), + "the put landed after the delete and put {} back", + hex::encode(addr) + ); + assert!( + !store.chunk_path(&addr).exists(), + "and left its file behind" ); - assert!(!store.legacy_only_keys().contains(&addr)); } - /// A delete outlasts a file write that no journal knows about. + /// A delete outlasts a write nobody waited for. + /// + /// A write's blocking half outlives the future that started it, deliberately, so the + /// work is never left half done. That means a cancelled put can still be queued when a + /// delete arrives, and if the delete does not wait for it the write lands afterwards + /// and puts back a chunk the node had decided to prune. The key is then in a store that + /// no longer claims it, which is what the next verification has to clean up. /// - /// The journal is kept by writes that touch both stores. The copier and the repair - /// path write only the file, so a delete that consulted the journal to decide whether - /// to wait would not wait for either of them, and whichever landed afterwards would - /// put back a chunk the node had decided to prune. What is writing a key is the file - /// store's own question to answer. + /// This ordering had a regression test before the migration facade was deleted, and the + /// test went with the facade even though the requirement did not. #[tokio::test] - async fn a_delete_outlasts_a_file_write_with_no_journal_entry() { + // The gate is held across awaits deliberately: holding it is what parks the put, which + // is the state the delete has to be ordered against. + #[allow(clippy::await_holding_lock)] + async fn a_delete_outlasts_a_write_nobody_waited_for() { let dir = TempDir::new().expect("temp dir"); - seed_legacy(&dir, &["neighbour"]).await; - let store = Arc::new(open(&dir).await); - let (addr, content) = addressed("copied-then-pruned"); - - let gate = store.files.test_put_gate(); - let (held_tx, held_rx) = std::sync::mpsc::channel::<()>(); - let (release_tx, release_rx) = std::sync::mpsc::channel::<()>(); - let holder = std::thread::spawn(move || { - let _parked = gate.write(); - held_tx.send(()).ok(); - release_rx.recv().ok(); - }); - held_rx.recv().expect("the gate is held"); - - // Deliberately no journal entry: this is the copier's shape, not a dual write. - let publishing = { - let files = Arc::clone(&store.files); + let store = Arc::new(reopen(&dir).await); + let content = b"a chunk that is about to be pruned".to_vec(); + let addr = crate::client::compute_address(&content); + + // Park the put inside its closure, then drop the future waiting on it. + let gate = store.test_put_gate(); + let held = gate.write(); + let put = { + let store = Arc::clone(&store); let content = content.clone(); - tokio::spawn(async move { files.put(&addr, &content).await }) + tokio::spawn(async move { store.put(&addr, &content).await }) }; - while store.files.tasks_in_flight() == 0 { + let deadline = std::time::Instant::now() + Duration::from_secs(30); + while store.tasks_in_flight() == 0 { + assert!( + std::time::Instant::now() < deadline, + "the put never reached the closure" + ); tokio::time::sleep(Duration::from_millis(5)).await; } - publishing.abort(); - let _ = publishing.await; - assert!( - !store - .legacy() - .is_some_and(|l| l.pending.read().contains_key(&addr)), - "this is the case the journal does not cover, so it must be empty" - ); + put.abort(); + let _ = put.await; + // The delete must not finish ahead of that write. Released after the delete has + // had time to be waiting, so if it does not wait, it wins the race and the test + // catches it. let deleting = { let store = Arc::clone(&store); tokio::spawn(async move { store.delete(&addr).await }) }; tokio::time::sleep(Duration::from_millis(100)).await; + drop(held); + let _ = deleting + .await + .expect("the delete task itself must not fail"); + store.wait_idle().await; + assert!( - !deleting.is_finished(), - "the delete must wait for a write the journal never knew about" + !store.is_indexed(&addr), + "the write landed after the delete and put {} back", + hex::encode(addr) ); - - release_tx.send(()).ok(); - holder.join().ok(); - deleting.await.expect("join").expect("delete"); - - store.files.wait_idle().await; assert!( - !store.exists(&addr).unwrap_or(true), - "a write that landed after the delete would resurrect a pruned chunk" + !store.chunk_path(&addr).exists(), + "and left its file on disk" ); } - /// A single node can still be told to keep both stores. + /// A put whose caller goes away does not admit a key on bytes nothing has read. + /// + /// The blocking half of a put outlives the future that started it, deliberately, so + /// the work is never left half done. That makes anything it writes to memory a claim + /// the node keeps whether or not the caller is still there to finish checking it. + /// + /// For a chunk this call published the claim is earned: the bytes were hashed against + /// their own name on the way in. For a name that was already taken it is not. The + /// check that decides whether those bytes are good runs after the await, and a dropped + /// future skips it, so admitting the key in the closure claims a chunk nobody read. + /// + /// Staged with a fifo, which is the sharpest case and a real one: the startup scan + /// refuses non-regular entries by design, so this is a key the store has already + /// decided it must not claim, walked in through the back door. + #[cfg(unix)] #[tokio::test] - async fn retirement_is_refused_when_the_switch_is_turned_off() { + // The gate is held across an await deliberately: holding it is what parks the put + // inside its closure, which is the state under test. Dropping it before awaiting would + // let the put finish and there would be nothing to cancel. + #[allow(clippy::await_holding_lock)] + async fn a_cancelled_put_does_not_admit_a_key_whose_bytes_were_never_read() { let dir = TempDir::new().expect("temp dir"); - seed_legacy(&dir, &["off"]).await; - let mut config = test_config(&dir); - config.migration.retire_legacy = false; - let store = ChunkStore::new(config).await.expect("open store"); - store.commit_to_files().expect("commit"); - assert!(store - .retirement_blocker(|_| false) - .expect("blocked") - .contains("retirement is disabled")); + let store = Arc::new(reopen(&dir).await); + + // A name a real chunk would use, wearing something that is not a chunk. + let content = b"the bytes that belong under this name".to_vec(); + let addr = crate::client::compute_address(&content); + let shard = dir + .path() + .join(CHUNKS_DIR_NAME) + .join(format!("{:02x}", addr[31])); + std::fs::create_dir_all(&shard).expect("mkdir"); + let path = shard.join(hex::encode(addr)); + let name = std::ffi::CString::new(path.as_os_str().as_encoded_bytes()) + .expect("a path with no interior nul"); + // SAFETY: `name` is a valid NUL-terminated C string that outlives the call, and the + // mode is a constant. `mkfifo` reads the pointer and returns; nothing is retained. + #[allow(clippy::undocumented_unsafe_blocks, unsafe_code)] + let made = unsafe { libc::mkfifo(name.as_ptr(), 0o644) }; + assert_eq!(made, 0, "could not make the fifo this test needs"); + + // Hold the gate so the put parks inside the closure, then drop the future while it + // is parked. That is a caller going away mid-put, which is what a cancelled + // request, a client disconnect or a shutdown all look like from in here. + let gate = store.test_put_gate(); + let held = gate.write(); + let put = { + let store = Arc::clone(&store); + let content = content.clone(); + tokio::spawn(async move { store.put(&addr, &content).await }) + }; + // Waited for rather than slept at. A sleep proves nothing: if the put had not + // reached the gated closure yet, aborting would cancel it before it ever got + // there and the test would pass having staged nothing. + let deadline = std::time::Instant::now() + Duration::from_secs(30); + while store.tasks_in_flight() == 0 { + assert!( + std::time::Instant::now() < deadline, + "the put never reached the closure, so there was nothing to cancel" + ); + tokio::time::sleep(Duration::from_millis(5)).await; + } + put.abort(); + let _ = put.await; + drop(held); + store.wait_idle().await; + + assert!( + !store.is_indexed(&addr), + "a cancelled put admitted {} on bytes nothing read; the fifo under that name \ + would then be advertised, committed to, and audited against", + hex::encode(addr) + ); + assert!( + !store.exists(&addr).unwrap_or(true), + "and the node must not claim it either" + ); } + /// A marker temporary left in the node root is swept, and nothing else is. + /// + /// The migration marker is written next to itself in the root, which no sweep looked + /// at, so a crash between its write and its rename left one there for the life of the + /// node. Small, but nothing was ever going to remove it. + /// + /// The second half is the point: this runs over a directory holding a node's data, so + /// it has to take only the exact shape this module writes and leave everything else + /// where it is. #[tokio::test] - async fn retiring_removes_the_legacy_environment_and_frees_its_space() { + async fn a_leftover_marker_temporary_is_swept_and_its_neighbours_are_not() { let dir = TempDir::new().expect("temp dir"); - let keys = seed_legacy(&dir, &["f1", "f2", "f3"]).await; - let mut config = test_config(&dir); - config.migration.retire_legacy = true; - let store = ChunkStore::new(config).await.expect("open"); - - store - .copy_batch(&keys, 0, 0, &never_cancelled()) - .await - .expect("copy"); - open_the_retirement_gate(&store); - - let proof = store - .verify_before_retire(0, &never_cancelled()) - .await - .expect("verify"); - assert!(proof.is_clean()); - assert_eq!(proof.checked, 3); + let root = dir.path(); + let leftover = root.join(format!("{TEMP_PREFIX}1234.abcdef01.marker")); + std::fs::write(&leftover, b"an interrupted marker write").expect("plant"); + + // Things that must survive: the marker itself, a chunk-shaped temp that belongs to + // the chunk tree's own sweep, and anything an operator put there. + let keep = [ + root.join("migration-state.json"), + root.join(format!("{TEMP_PREFIX}1234.abcdef01.chunk")), + root.join("notes.txt"), + // Prefix and suffix alone would take these. The pid and the nonce are checked + // because this runs over a directory holding a node's data. + root.join(format!("{TEMP_PREFIX}operator-notes.marker")), + root.join(format!("{TEMP_PREFIX}1234.nothex01.marker")), + root.join(format!("{TEMP_PREFIX}1234.abcdef0.marker")), + root.join(format!("{TEMP_PREFIX}1234.abcdef01.extra.marker")), + ]; + for path in &keep { + std::fs::write(path, b"keep me").expect("plant"); + } + + let store = reopen(&dir).await; + drop(store); - let freed = store - .retire_legacy(&proof, &|_: &XorName| false, &approved_shed(&store)) - .await - .expect("retire"); - assert!(freed > 0, "retirement must report the space it returned"); assert!( - !dir.path().join(LEGACY_ENV_DIR).exists(), - "the legacy environment must actually be removed" + !leftover.exists(), + "the leftover marker temporary is still in the node root" ); - assert!(!store.has_legacy()); - assert_eq!(store.migration_phase(), MigrationPhase::FilesOnly); - - // Everything is still readable, from files alone. - assert_eq!(store.current_chunks().expect("count"), 3); - for key in &keys { - assert!(store.get(key).await.expect("get").is_some()); + for path in &keep { + assert!( + path.exists(), + "{} was swept and should not have been", + path.display() + ); } } #[tokio::test] - async fn a_reader_holding_the_legacy_handle_defers_retirement_without_hiding_chunks() { - let dir = TempDir::new().expect("temp dir"); - let keys = seed_legacy(&dir, &["busy"]).await; - let mut config = test_config(&dir); - config.migration.retire_legacy = true; - let store = Arc::new(ChunkStore::new(config).await.expect("open")); - store - .copy_batch(&keys, 0, 0, &never_cancelled()) - .await - .expect("copy"); - open_the_retirement_gate(&store); - - // Stand in for a read that is still holding the legacy handle. Retirement must - // defer rather than unmap underneath it, and the chunk must stay readable - // throughout: a retirement attempt that briefly hid the legacy store would make a - // node answer "not found" for a chunk it holds. - let squatter = store.legacy().expect("a legacy handle"); - let proof = store - .verify_before_retire(0, &never_cancelled()) - .await - .expect("verify"); - let err = store - .retire_legacy(&proof, &|_: &XorName| false, &approved_shed(&store)) - .await - .expect_err("must defer while the handle is held"); - assert!(format!("{err}").contains("deferred"), "{err}"); + async fn a_corrupt_chunk_is_removed_so_replication_can_repair_it() { + let (store, _dir) = test_store().await; + let (addr, content) = addressed("corrupt"); + store.put(&addr, &content).await.expect("put"); + std::fs::write(store.chunk_path(&addr), b"tampered").expect("tamper"); - assert!(store.has_legacy(), "the store must keep its legacy handle"); - let key = keys.first().copied().expect("a key"); + let err = store.get(&addr).await.expect_err("verification must fail"); + assert!(format!("{err}").contains("verification failed"), "{err}"); + + assert!(!store.chunk_path(&addr).exists(), "corrupt file must go"); + assert!(!store.exists(&addr).expect("exists")); assert!( - store.get(&key).await.expect("get").is_some(), - "the chunk must stay readable across a deferred retirement" + !store.all_keys().await.expect("all_keys").contains(&addr), + "a corrupt chunk must stop being advertised" ); - - // Once the reader lets go, the next attempt succeeds. - drop(squatter); - store - .retire_legacy(&proof, &|_: &XorName| false, &approved_shed(&store)) - .await - .expect("retire"); - assert!(!store.has_legacy()); + assert_eq!(store.stats().verification_failures, 1); } #[tokio::test] - async fn retirement_needs_a_clean_verification_report() { - let dir = TempDir::new().expect("temp dir"); - // Left uncopied on purpose, so this node is about to give the chunk up and the - // answerability veto has something to fire on. - seed_legacy(&dir, &["v1"]).await; - let mut config = test_config(&dir); - config.migration.retire_legacy = true; - let store = ChunkStore::new(config).await.expect("open"); - - // A report cannot be fabricated: every field is private and the only source is - // the verification pass. The one available here is the default, which never ran. - let absent = VerifyReport::default(); - let err = store - .retire_legacy(&absent, &|_: &XorName| false, &approved_shed(&store)) - .await - .expect_err("must refuse a report that never ran"); - assert!(format!("{err}").contains("unrepairable"), "{err}"); - assert!(store.has_legacy(), "the legacy environment must survive"); - - // And a real, clean report is still refused while any gate is unmet, because - // retirement rechecks them all itself rather than trusting its caller. - open_the_retirement_gate(&store); - let proof = store - .verify_before_retire(0, &never_cancelled()) - .await - .expect("verify"); - assert!(proof.is_clean()); - let err = store - .retire_legacy(&proof, &|_: &XorName| true, &approved_shed(&store)) - .await - .expect_err("must refuse while a chunk is still answerable"); - assert!(format!("{err}").contains("still answerable"), "{err}"); - assert!(store.has_legacy()); + async fn a_file_removed_underneath_the_store_drops_out_of_the_index() { + let (store, _dir) = test_store().await; + let (addr, content) = addressed("vanished"); + store.put(&addr, &content).await.expect("put"); + + std::fs::remove_file(store.chunk_path(&addr)).expect("remove behind our back"); + + assert!(store.get(&addr).await.expect("get").is_none()); + assert!(!store.exists(&addr).expect("exists")); + assert_eq!(store.current_chunks().expect("count"), 0); } #[tokio::test] - async fn verification_repairs_a_file_that_rotted_before_retirement() { - let dir = TempDir::new().expect("temp dir"); - let keys = seed_legacy(&dir, &["w1", "w2"]).await; - let mut config = test_config(&dir); - config.migration.retire_legacy = true; - let store = ChunkStore::new(config).await.expect("open"); - store - .copy_batch(&keys, 0, 0, &never_cancelled()) - .await - .expect("copy"); + async fn interrupted_writes_are_swept_at_startup() { + let (store, dir) = test_store().await; + let (addr, content) = addressed("sweep"); + store.put(&addr, &content).await.expect("put"); + let shard = store + .chunk_path(&addr) + .parent() + .expect("shard") + .to_path_buf(); + drop(store); - // A filename is not proof the bytes behind it are good. Corrupt one, exactly as - // a truncated write or a bad sector would, then prove retirement repairs it - // rather than deleting the only intact copy. - let victim = keys.first().copied().expect("a key"); - let path = dir + let orphan = shard.join(format!("{TEMP_PREFIX}999.7")); + std::fs::write(&orphan, b"half a chunk").expect("write orphan"); + let stray_root = dir .path() - .join(crate::storage::file_store::CHUNKS_DIR_NAME) - .join(format!("{:02x}", victim.last().copied().unwrap_or(0))) - .join(hex::encode(victim)); - std::fs::write(&path, b"rotted").expect("corrupt the file"); - open_the_retirement_gate(&store); - - let proof = store - .verify_before_retire(0, &never_cancelled()) - .await - .expect("verify"); - assert_eq!(proof.repaired(), 1); - assert_eq!(proof.unrepairable(), 0); - assert!(proof.is_clean()); + .join(CHUNKS_DIR_NAME) + .join(format!("{TEMP_PREFIX}999.8")); + std::fs::write(&stray_root, b"half a marker").expect("write stray"); - store - .retire_legacy(&proof, &|_: &XorName| false, &approved_shed(&store)) - .await - .expect("retire"); - assert_eq!( - store.get(&victim).await.expect("get").expect("present"), - addressed("w1").1 - ); + let reopened = reopen(&dir).await; + assert!(!orphan.exists(), "an interrupted write must not survive"); + assert!(!stray_root.exists(), "nor one at the store root"); + assert_eq!(reopened.current_chunks().expect("count"), 1); } #[tokio::test] - async fn a_corrupt_file_is_served_from_the_legacy_copy_and_requeued() { - let dir = TempDir::new().expect("temp dir"); - let keys = seed_legacy(&dir, &["s1"]).await; - let store = open(&dir).await; - store - .copy_batch(&keys, 0, 0, &never_cancelled()) - .await - .expect("copy"); - assert!(store.legacy_only_keys().is_empty()); + async fn concurrent_writers_of_one_address_store_it_exactly_once() { + let (store, _dir) = test_store().await; + let store = Arc::new(store); + let (addr, content) = addressed("racing"); - let victim = keys.first().copied().expect("a key"); - let path = dir - .path() - .join(crate::storage::file_store::CHUNKS_DIR_NAME) - .join(format!("{:02x}", victim.last().copied().unwrap_or(0))) - .join(hex::encode(victim)); - std::fs::write(&path, b"rotted").expect("corrupt the file"); + let mut tasks = Vec::new(); + for _ in 0..16 { + let store = Arc::clone(&store); + let content = content.clone(); + tasks.push(tokio::spawn( + async move { store.put(&addr, &content).await }, + )); + } - // The file store removes the bad file and stops advertising it; the facade must - // still find the intact copy rather than reporting a failure. + let mut new_count = 0; + for task in tasks { + if task.await.expect("join").expect("put") { + new_count += 1; + } + } + assert_eq!(new_count, 1, "exactly one writer may report a new chunk"); + assert_eq!(store.current_chunks().expect("count"), 1); assert_eq!( - store.get(&victim).await.expect("get").expect("present"), - addressed("s1").1 - ); - assert!( - store.legacy_only_keys().contains(&victim), - "the key must go back on the copier's list" + store.get(&addr).await.expect("get").expect("present"), + content ); } #[tokio::test] - async fn a_marker_claiming_more_than_the_file_store_holds_restarts_the_copy() { - let dir = TempDir::new().expect("temp dir"); - let keys = seed_legacy(&dir, &["p1", "p2", "p3"]).await; - let store = open(&dir).await; - store - .copy_batch(&keys, 0, 0, &never_cancelled()) - .await - .expect("copy"); - store.commit_to_files().expect("commit"); - assert_eq!(store.migration_state().kept_key_count, 3); - store.wait_idle().await; + async fn names_that_are_not_lowercase_hex_are_ignored_by_the_scan() { + let (store, dir) = test_store().await; + let (addr, content) = addressed("scan"); + store.put(&addr, &content).await.expect("put"); + let shard = store + .chunk_path(&addr) + .parent() + .expect("shard") + .to_path_buf(); drop(store); - // Someone clears the chunk directory to reclaim space, keeping chunks.mdb. - // Trusting the marker here would skip the copier, the shed rules and their rank - // checks on the way to deleting the legacy environment. - let chunks = dir.path().join(crate::storage::file_store::CHUNKS_DIR_NAME); - for key in &keys { - let path = chunks - .join(format!("{:02x}", key.last().copied().unwrap_or(0))) - .join(hex::encode(key)); - std::fs::remove_file(path).expect("clear the file store"); - } + // Uppercase is deliberately rejected: on a case-folding filesystem accepting it + // would let one file answer to two index entries. + let upper = shard.join(hex::encode_upper(addressed("upper").0)); + std::fs::write(&upper, b"x").expect("write upper"); + std::fs::write(shard.join("not-a-chunk"), b"x").expect("write junk"); + std::fs::write(shard.join("deadbeef"), b"x").expect("write short"); - let store = open(&dir).await; - assert_eq!( - store.migration_phase(), - MigrationPhase::Bridging, - "the filesystem must win over the marker" - ); - assert_eq!(store.legacy_only_keys().len(), 3); + let reopened = reopen(&dir).await; + assert_eq!(reopened.current_chunks().expect("count"), 1); } #[tokio::test] - async fn a_file_that_vanished_mid_verification_is_requeued_not_republished() { - let dir = TempDir::new().expect("temp dir"); - let keys = seed_legacy(&dir, &["gone"]).await; - let mut config = test_config(&dir); - config.migration.retire_legacy = true; - let store = ChunkStore::new(config).await.expect("open"); - store - .copy_batch(&keys, 0, 0, &never_cancelled()) - .await - .expect("copy"); - assert!(store.legacy_only_keys().is_empty()); - - // Remove the file without telling the store, which is what the pruner's own - // delete looks like if it lands mid-pass. Republishing from the legacy copy here - // would resurrect a chunk the node had deliberately deleted, so the key goes back - // on the copier's list instead and retirement is refused. - let key = keys.first().copied().expect("a key"); - let path = dir + async fn a_chunk_filed_in_the_wrong_shard_is_not_indexed() { + let (store, dir) = test_store().await; + let (addr, content) = addressed("misfiled"); + store.put(&addr, &content).await.expect("put"); + drop(store); + + // Move it one shard over: the read path would never find it there, so indexing + // it would make the store advertise a key it cannot serve. + let correct = dir + .path() + .join(CHUNKS_DIR_NAME) + .join(shard_name(&addr)) + .join(hex::encode(addr)); + let wrong_shard_index = (shard_index(&addr) + 1) % SHARD_COUNT; + let wrong_dir = dir .path() - .join(crate::storage::file_store::CHUNKS_DIR_NAME) - .join(format!("{:02x}", key.last().copied().unwrap_or(0))) - .join(hex::encode(key)); - std::fs::remove_file(&path).expect("remove behind the store's back"); + .join(CHUNKS_DIR_NAME) + .join(format!("{wrong_shard_index:02x}")); + std::fs::create_dir_all(&wrong_dir).expect("mkdir"); + std::fs::rename(&correct, wrong_dir.join(hex::encode(addr))).expect("misfile"); - let proof = store - .verify_before_retire(0, &never_cancelled()) - .await - .expect("verify"); - assert!( - proof.unrepairable >= 1, - "the vanished file must be counted against the proof" - ); - assert!(!proof.is_clean()); - assert!(!path.exists(), "the pass must not republish it"); - assert!(store.legacy_only_keys().contains(&key)); - assert!(store - .retire_legacy(&proof, &|_: &XorName| false, &approved_shed(&store)) - .await - .is_err()); - assert!(store.has_legacy()); + let reopened = reopen(&dir).await; + assert_eq!(reopened.current_chunks().expect("count"), 0); + assert!(!reopened.exists(&addr).expect("exists")); } #[tokio::test] - async fn a_cancelled_shutdown_stops_the_copier_and_refuses_to_pass_verification() { - let dir = TempDir::new().expect("temp dir"); - let keys = seed_legacy(&dir, &["s1", "s2", "s3"]).await; - let store = open(&dir).await; + async fn the_layout_marker_is_written_once_and_checked_on_reopen() { + let (store, dir) = test_store().await; + drop(store); - // Shutdown must not have to wait out a pass that can run for hours, and a pass - // that stopped early is not evidence of anything. - let cancelled = CancellationToken::new(); - cancelled.cancel(); + let marker = dir.path().join(CHUNKS_DIR_NAME).join(LAYOUT_FILE_NAME); + let layout: StoreLayout = + serde_json::from_slice(&std::fs::read(&marker).expect("read marker")) + .expect("parse marker"); + assert_eq!(layout, StoreLayout::default()); - let report = store - .copy_batch(&keys, 0, 0, &cancelled) - .await - .expect("copy"); - assert_eq!(report.copied, 0, "the copier must stop immediately"); - assert_eq!(store.legacy_only_keys().len(), 3); + // A store written by a future build must be refused, not misread. + let future = StoreLayout { + schema: LAYOUT_SCHEMA + 1, + ..StoreLayout::default() + }; + std::fs::write(&marker, serde_json::to_vec(&future).expect("encode")).expect("write"); + let err = ChunkStore::new(ChunkStoreConfig { + root_dir: dir.path().to_path_buf(), + verify_on_read: true, + disk_reserve: 0, + }) + .await + .expect_err("must refuse a newer layout"); + assert!(format!("{err}").contains("newer than this build"), "{err}"); + } - let proof = store - .verify_before_retire(0, &cancelled) - .await - .expect("verify"); + #[tokio::test] + async fn an_unknown_shard_scheme_is_refused() { + let (store, dir) = test_store().await; + drop(store); + let marker = dir.path().join(CHUNKS_DIR_NAME).join(LAYOUT_FILE_NAME); + let other = StoreLayout { + scheme: "prefix-hex".to_string(), + ..StoreLayout::default() + }; + std::fs::write(&marker, serde_json::to_vec(&other).expect("encode")).expect("write"); + let err = ChunkStore::new(ChunkStoreConfig { + root_dir: dir.path().to_path_buf(), + verify_on_read: true, + disk_reserve: 0, + }) + .await + .expect_err("must refuse an unknown scheme"); + assert!(format!("{err}").contains("shard scheme"), "{err}"); + } + + #[tokio::test] + async fn writes_are_refused_when_the_disk_reserve_cannot_be_met() { + let dir = TempDir::new().expect("temp dir"); + let store = ChunkStore::new(ChunkStoreConfig { + root_dir: dir.path().to_path_buf(), + verify_on_read: true, + disk_reserve: u64::MAX / 2, + }) + .await + .expect("open store"); + + let (addr, content) = addressed("full"); + let err = store.put(&addr, &content).await.expect_err("must refuse"); assert!( - !proof.is_clean(), - "an interrupted verification must never read as a pass" + format!("{err}").contains("Insufficient disk space"), + "{err}" ); + assert!(store.check_capacity().is_err()); } #[tokio::test] - async fn the_migration_marker_survives_a_restart() { + async fn capacity_is_size_aware() { + // Wide enough that a test running alongside this one cannot move the answer. + const MARGIN: u64 = 512 * 1024 * 1024; + let dir = TempDir::new().expect("temp dir"); - let keys = seed_legacy(&dir, &["m1", "m2"]).await; - let store = open(&dir).await; - store - .copy_batch(&keys[..1], 0, 0, &never_cancelled()) - .await - .expect("copy"); - store.commit_to_files().expect("commit"); - store.note_commitment_rebuilt(); - let first_start = store.migration_state().first_start_unix; - store.wait_idle().await; - drop(store); + let available = fs2::available_space(dir.path()).expect("free space"); + // A reserve that leaves room for a small write but not a huge one. This is the + // whole reason the predicate takes a size: free bytes alone stopped being a + // sufficient answer once chunks became files. + let store = ChunkStore::new(ChunkStoreConfig { + root_dir: dir.path().to_path_buf(), + verify_on_read: true, + disk_reserve: available.saturating_sub(MARGIN), + }) + .await + .expect("open store"); + + assert!(store.check_capacity_for(1024).is_ok()); + assert!(store.check_capacity_for(4 * MARGIN).is_err()); + } - let store = open(&dir).await; - let state = store.migration_state(); - assert_eq!(state.phase, MigrationPhase::Committed); - assert_eq!(state.shed_key_count, 1); + #[test] + fn suffix_shards_stay_uniform_for_a_close_group_of_keys() { + // The real distribution: a node holds keys it is closest to, so they share a + // long leading prefix with its own ID. Sharding on that prefix collapses to one + // directory. The trailing byte is untouched by close-group membership. + let mut prefix_dirs = HashSet::new(); + let mut suffix_dirs = HashSet::new(); + for i in 0u32..4096 { + let mut key = [0u8; XORNAME_LEN]; + // 20 shared leading bits, as a ~1M-node network would impose. + let tail = crate::client::compute_address(&i.to_le_bytes()); + key.copy_from_slice(&tail); + if let Some(b) = key.first_mut() { + *b = 0xab; + } + if let Some(b) = key.get_mut(1) { + *b = 0xcd; + } + if let Some(b) = key.get_mut(2) { + *b &= 0x0f; + } + prefix_dirs.insert(key.first().copied().unwrap_or(0)); + suffix_dirs.insert(shard_index(&key)); + } assert_eq!( - state.first_start_unix, first_start, - "a restart must not restart the shed hold" + prefix_dirs.len(), + 1, + "prefix sharding collapses for a node's own holdings" ); assert!( - state.committed_at_unix.is_some(), - "nor the retirement clock" + suffix_dirs.len() > 250, + "suffix sharding must stay uniform, got {} of 256 directories", + suffix_dirs.len() ); } + #[test] + fn no_chunk_filename_can_spell_a_reserved_windows_device_name() { + // Hex has no `n`, `u`, `x`, `p`, `r`, `l`, `t`, `o` or `s`, so `CON`, `NUL`, + // `AUX`, `PRN`, `COM1` and `LPT1` are all unspellable at any length. This is why + // the encoding is hex and not base32 or base64url. + for reserved in ["con", "prn", "aux", "nul", "com1", "com9", "lpt1", "lpt9"] { + assert!( + !is_lower_hex(reserved), + "{reserved} must not be a valid chunk or shard name" + ); + } + } + + #[test] + fn only_full_length_lowercase_hex_decodes_to_an_address() { + // 0xab so the hex form actually contains letters, which is where case matters. + assert!(decode_chunk_name(&hex::encode([0xabu8; XORNAME_LEN])).is_some()); + assert!(decode_chunk_name(&hex::encode_upper([0xabu8; XORNAME_LEN])).is_none()); + assert!(decode_chunk_name("deadbeef").is_none()); + assert!(decode_chunk_name("").is_none()); + assert!(decode_chunk_name(&"g".repeat(CHUNK_NAME_LEN)).is_none()); + } + #[tokio::test] - async fn a_marker_that_disagrees_with_the_filesystem_loses() { - let dir = TempDir::new().expect("temp dir"); - seed_legacy(&dir, &["x1"]).await; - let store = open(&dir).await; - store.force_migration_state(|s| s.phase = MigrationPhase::FilesOnly); - store.migration_state().save(dir.path()).expect("save"); - store.wait_idle().await; - drop(store); + async fn repair_replaces_bad_bytes_without_the_file_ever_being_absent() { + let (store, _dir) = test_store().await; + let (addr, content) = addressed("repairable"); + store.put(&addr, &content).await.expect("put"); + let path = store.chunk_path(&addr); + + std::fs::write(&path, b"rotted").expect("corrupt"); + store.repair(&addr, &content).await.expect("repair"); - // The marker claims the migration is done, but `chunks.mdb` is right there. The - // filesystem is the authority. - let store = open(&dir).await; - assert_eq!(store.migration_phase(), MigrationPhase::Bridging); - assert!(store.has_legacy()); + assert_eq!( + store.get(&addr).await.expect("get").expect("present"), + content + ); + assert!(store.exists(&addr).expect("exists")); } - /// A legacy record whose bytes do not hash to its key is removed, not passed around. - /// - /// Leaving it in the environment while dropping it from the key set puts it in - /// neither view, and the pre-retirement pass reads a key in neither view as one to - /// protect and puts it straight back. The next copier pass drops it again. One rotted - /// record would keep this node, and every node sharing its disk, from ever reclaiming - /// space. #[tokio::test] - async fn a_legacy_chunk_that_does_not_match_its_address_is_removed_not_recycled() { - let dir = TempDir::new().expect("temp dir"); - let (addr, _) = addressed("bad"); - let other = addressed("other").1; - { - let lmdb = LmdbStorage::new(LmdbStorageConfig { - root_dir: dir.path().to_path_buf(), - verify_on_read: false, - max_map_size: 0, - disk_reserve: 0, - }) - .await - .expect("open legacy"); - // Under a key it does not hash to: what a record that rotted in place looks - // like, and the one shape the ordinary path refuses to create. - lmdb.put_unchecked(&addr, &other).await.expect("put"); - lmdb.wait_idle().await; - } - let store = open(&dir).await; - let keys = store.legacy_only_keys(); - assert_eq!(keys, vec![addr]); + async fn a_repair_with_the_wrong_bytes_is_refused_and_changes_nothing() { + let (store, _dir) = test_store().await; + let (addr, content) = addressed("guarded"); + store.put(&addr, &content).await.expect("put"); + let path = store.chunk_path(&addr); - let report = store - .copy_batch(&keys, 0, 0, &never_cancelled()) - .await - .expect("copy"); - assert_eq!(report.copied, 0); - assert_eq!(report.unusable, 1); - assert!(store.legacy_only_keys().is_empty()); - - // And it is gone from the environment, so the pass below cannot find it and put - // it back. That is the loop this is about. - let proof = store - .verify_before_retire(0, &never_cancelled()) + // The whole point of repairing in place is that a failure must leave the old file + // where it was. Deleting first and writing after would open a window whose only + // surviving copy is the one the caller is about to destroy. + let err = store + .repair(&addr, b"not this chunk") .await - .expect("verify"); + .expect_err("must refuse"); + assert!(format!("{err}").contains("Refusing to repair"), "{err}"); assert!( - store.legacy_only_keys().is_empty(), - "a removed record must not come back on the copier's list" + path.exists(), + "the existing file must survive a refused repair" ); + assert_eq!( + store.get(&addr).await.expect("get").expect("present"), + content + ); + } + + #[tokio::test] + async fn a_chunk_can_be_deleted_and_stored_again() { + let (store, _dir) = test_store().await; + let (addr, content) = addressed("cycle"); + + assert!(store.put(&addr, &content).await.expect("put")); + assert!(store.delete(&addr).await.expect("delete")); assert!( - proof.is_clean(), - "and must not go on refusing the proof for ever" + store.put(&addr, &content).await.expect("re-put"), + "a re-stored chunk is new again" + ); + assert_eq!( + store.get(&addr).await.expect("get").expect("present"), + content ); } - #[test] - fn the_release_switches_are_never_written_to_an_operator_config_file() { - // A node writes its effective configuration back to disk. If these round-tripped, - // R1's values would be baked into every operator's file and the next release - // would change nothing. - let mut config = MigrationConfig::default(); - config.retire_legacy = !config.retire_legacy; - config.allow_shed = false; - config.shed_hold_hours = 5; - - let encoded = toml::to_string(&config).expect("encode"); - assert!(!encoded.contains("retire_legacy"), "{encoded}"); - - let decoded: MigrationConfig = toml::from_str(&encoded).expect("decode"); - let fresh = MigrationConfig::default(); - assert_eq!(decoded.retire_legacy, fresh.retire_legacy); - // Genuine operator controls do survive. - assert!(!decoded.allow_shed); - assert_eq!(decoded.shed_hold_hours, 5); + /// Write a chunk file straight into its shard, the way an existing store already + /// contains thousands of them. Bypasses the write path deliberately: this exercises + /// the startup scan, not `put`. + fn plant(chunks_dir: &Path, key: &XorName) { + let dir = chunks_dir.join(shard_name(key)); + std::fs::create_dir_all(&dir).expect("mkdir"); + std::fs::write(dir.join(hex::encode(key)), key).expect("plant"); } - #[test] - fn the_copy_order_is_closest_first() { - let me = [0u8; XORNAME_LEN_LOCAL]; - let mut near = [0u8; XORNAME_LEN_LOCAL]; - if let Some(b) = near.last_mut() { - *b = 1; - } - let mut far = [0u8; XORNAME_LEN_LOCAL]; - if let Some(b) = far.first_mut() { - *b = 0xff; - } - let ordered = rank_closest_first(vec![far, near], Some(me)); - assert_eq!(ordered.first().copied(), Some(near)); - assert_eq!(ordered.last().copied(), Some(far)); + #[tokio::test] + async fn a_populated_and_churned_store_scans_correctly_at_scale() { + // Every shard populated, then aged the way a long-lived node ages: some keys + // deleted, others added in their place, so the directories carry holes rather + // than being freshly written. APFS enumeration is known to degrade with churn + // rather than with size, so a fresh corpus is not a realistic one. + const PLANTED: u32 = 20_000; + const CHURN: u32 = 1_000; - // With no identity the order is still stable, which is all the copier needs. - let ordered = rank_closest_first(vec![far, near], None); - let mut expected = vec![far, near]; + let dir = TempDir::new().expect("temp dir"); + let chunks_dir = dir.path().join(CHUNKS_DIR_NAME); + std::fs::create_dir_all(&chunks_dir).expect("mkdir"); + + let mut expected: Vec = Vec::new(); + for i in 0..PLANTED { + let key = crate::client::compute_address(&i.to_le_bytes()); + plant(&chunks_dir, &key); + expected.push(key); + } + for i in 0..CHURN { + let key = crate::client::compute_address(&i.to_le_bytes()); + std::fs::remove_file(chunks_dir.join(shard_name(&key)).join(hex::encode(key))) + .expect("churn out"); + let replacement = crate::client::compute_address(&(PLANTED + i).to_le_bytes()); + plant(&chunks_dir, &replacement); + } + expected.retain(|k| chunks_dir.join(shard_name(k)).join(hex::encode(k)).exists()); + for i in 0..CHURN { + expected.push(crate::client::compute_address(&(PLANTED + i).to_le_bytes())); + } expected.sort_unstable(); - assert_eq!(ordered, expected); - } + expected.dedup(); - /// Local alias so the test does not import from the protocol crate. - const XORNAME_LEN_LOCAL: usize = 32; + let started = std::time::Instant::now(); + let store = reopen(&dir).await; + let scan = started.elapsed(); - #[test] - fn the_retirement_delay_can_never_be_shortened_below_the_retention_window() { - let config = MigrationConfig { - retire_delay_hours: 0, - ..MigrationConfig::default() - }; assert_eq!( - config.effective_retire_delay_hours(), - MIN_RETIRE_DELAY_HOURS + store.current_chunks().expect("count"), + expected.len() as u64 + ); + assert_eq!(store.all_keys().await.expect("all_keys"), expected); + + // Every shard should be in use at this size: 20,000 keys over 256 directories is + // about 78 each, and the last byte of a BLAKE3 output is uniform. + let occupied = std::fs::read_dir(&chunks_dir) + .expect("read store root") + .filter_map(std::result::Result::ok) + .filter(|e| e.file_name().to_str().is_some_and(|n| n.len() == 2)) + .count(); + assert_eq!(occupied, SHARD_COUNT, "the suffix must reach every shard"); + + println!( + "scan of {} keys across {SHARD_COUNT} shards took {scan:?}", + expected.len() ); } + + #[tokio::test] + async fn wait_idle_returns_once_writes_have_drained() { + let (store, _dir) = test_store().await; + let store = Arc::new(store); + for i in 0..32 { + let store = Arc::clone(&store); + let (addr, content) = addressed(&format!("drain-{i}")); + tokio::spawn(async move { store.put(&addr, &content).await }); + } + // Not a synchronisation point for tasks that have not been spawned yet, but it + // must not hang and it must leave the store usable. + store.wait_idle().await; + let (addr, content) = addressed("after-drain"); + assert!(store.put(&addr, &content).await.expect("put after drain")); + } } diff --git a/src/storage/file_store.rs b/src/storage/file_store.rs deleted file mode 100644 index 327900dd..00000000 --- a/src/storage/file_store.rs +++ /dev/null @@ -1,3706 +0,0 @@ -//! One immutable file per chunk, content-addressed, with the filesystem as the -//! only authority. -//! -//! ```text -//! {root}/chunks/ store root -//! {root}/chunks/layout.json versioned layout marker -//! {root}/chunks/.lock advisory single-process guard -//! {root}/chunks//<64-hex> xy = the LAST two hex characters of the address -//! {root}/chunks//.tmp.. an in-flight write, in the destination directory -//! ``` -//! -//! # Why the *last* two hex characters -//! -//! A node holds keys for which it is among the [`CLOSE_GROUP_SIZE`] closest, so its -//! holdings share roughly `log2(N / CLOSE_GROUP_SIZE)` leading bits with its own node -//! ID, and that shared prefix grows as the network grows. Sharding on a prefix therefore -//! does not degrade, it collapses: at ~800 nodes a two-hex prefix already resolves to -//! about two distinct directories, and past a million nodes even a four-hex prefix -//! resolves to one. Close-group membership constrains the leading bits and places no -//! constraint at all on the trailing ones, and the address is a BLAKE3 output, so the -//! last byte is uniform by construction at every network size. -//! -//! 256 shards keeps a 24 GiB node at ~23 files per directory and a 1 TiB node at ~977, -//! for 1 MiB of directory inodes. The scheme and depth are recorded in `layout.json` at -//! creation so a future layout can be detected rather than silently misread. -//! -//! # Why lowercase hex names -//! -//! NTFS and default APFS fold case. Under an encoding with both cases (base64url, -//! base58) two distinct 32-byte keys can share one case-folded filename, which is a -//! silent overwrite. Hex has one case-folded form per key, and no hex string can ever -//! spell a reserved Windows device name (`CON`, `NUL`, `AUX`, `COM1`, ...) because none -//! of those letters is in `0-9a-f`. The full 64-character key stays in the filename, so -//! a `find` over the tree recovers the whole store even if the directory layer is lost. -//! -//! [`CLOSE_GROUP_SIZE`]: crate::ant_protocol::CLOSE_GROUP_SIZE - -use crate::ant_protocol::{XorName, MAX_CHUNK_SIZE, XORNAME_LEN}; -use crate::error::{Error, Result}; -use crate::logging::{debug, info, trace, warn}; -use crate::storage::StorageStats; -use fs2::FileExt; -use serde::{Deserialize, Serialize}; -use std::collections::{BTreeSet, HashMap, HashSet}; -use std::fs::{File, OpenOptions}; -use std::io::{ErrorKind, Read, Write}; -use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::Arc; -use std::time::{Duration, Instant}; -use tokio::task::spawn_blocking; -use tokio_util::task::TaskTracker; - -/// Directory under the node root that holds the chunk files. -pub const CHUNKS_DIR_NAME: &str = "chunks"; - -/// Name of the layout marker written once at store creation. -pub const LAYOUT_FILE_NAME: &str = "layout.json"; - -/// Name of the advisory single-process lock file. -const LOCK_FILE_NAME: &str = ".lock"; - -/// Prefix that marks an in-flight write. Never a valid chunk name (chunk names are -/// exactly [`CHUNK_NAME_LEN`] lowercase hex characters, and `.` is not hex). -const TEMP_PREFIX: &str = ".tmp."; - -/// Number of shard directories. One level, `00` through `ff`. -const SHARD_COUNT: usize = 256; - -/// Length of a chunk filename: the full address in lowercase hex. -const CHUNK_NAME_LEN: usize = XORNAME_LEN * 2; - -/// How often to re-query available disk space, in seconds. -/// -/// Matches the LMDB store's cadence so the capacity predicate behaves identically -/// for callers that only ask "is there room at all". -const DISK_CHECK_INTERVAL_SECS: u64 = 5; - -/// Allocation granularity assumed when charging a pending write against free space. -/// -/// Every filesystem we support allocates in units of at least 4 KiB, so a write of -/// `n` bytes consumes at least `ceil(n / 4096) * 4096`. One extra unit covers the -/// directory entry and inode. -const ALLOC_UNIT: u64 = 4096; - -/// How many times a publish retries a transient Windows sharing violation. -const RENAME_RETRY_ATTEMPTS: u32 = 5; - -/// Base backoff between those retries; the wait grows linearly with the attempt. -const RENAME_RETRY_BACKOFF: Duration = Duration::from_millis(20); - -/// Longest absolute path a chunk file may need, checked once at open. -/// -/// Windows caps a non-verbatim path at `MAX_PATH` (260) including the terminating NUL. -/// Rust's standard library transparently switches to the `\\?\` verbatim form for long -/// absolute paths, so this is a warning rather than a hard failure, but an operator who -/// buries the node root ten directories deep should hear about it before the first write -/// fails rather than after. -#[cfg(windows)] -const WINDOWS_PATH_WARN_LEN: usize = 240; - -/// The on-disk layout marker. -/// -/// Written once when the store directory is created and read on every subsequent open. -/// Nothing in this survey of comparable stores (IPFS flatfs, Storj, borgbackup) shipped -/// an in-place re-sharder, and all three paid for it. Recording the scheme costs one -/// small file and is the difference between changing the default later and never being -/// able to. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct StoreLayout { - /// Marker schema version. A store written by a newer schema is refused. - pub schema: u32, - /// How a chunk address maps to a shard directory. - pub scheme: String, - /// How many hex characters of the address name the shard directory. - pub shard_chars: u8, - /// How many directory levels of sharding. - pub depth: u8, - /// How a chunk address maps to a filename. - pub name_encoding: String, -} - -/// Marker schema this build writes and understands. -const LAYOUT_SCHEMA: u32 = 1; -/// Shard scheme this build implements: the trailing hex characters of the address. -const LAYOUT_SCHEME_SUFFIX_HEX: &str = "suffix-hex"; -/// Filename encoding this build implements. -const LAYOUT_NAME_LOWER_HEX: &str = "lower-hex"; - -impl Default for StoreLayout { - fn default() -> Self { - Self { - schema: LAYOUT_SCHEMA, - scheme: LAYOUT_SCHEME_SUFFIX_HEX.to_string(), - shard_chars: 2, - depth: 1, - name_encoding: LAYOUT_NAME_LOWER_HEX.to_string(), - } - } -} - -impl StoreLayout { - /// Return an error unless this build can read a store written with this layout. - fn check_supported(&self) -> Result<()> { - if self.schema > LAYOUT_SCHEMA { - return Err(Error::Storage(format!( - "Chunk store layout schema {} is newer than this build understands ({LAYOUT_SCHEMA}). \ - Refusing to open rather than misread the store.", - self.schema - ))); - } - if self.scheme != LAYOUT_SCHEME_SUFFIX_HEX { - return Err(Error::Storage(format!( - "Chunk store uses shard scheme '{}', this build implements '{LAYOUT_SCHEME_SUFFIX_HEX}'", - self.scheme - ))); - } - if self.shard_chars != 2 || self.depth != 1 { - return Err(Error::Storage(format!( - "Chunk store uses {} shard characters at depth {}, this build implements 2 at depth 1", - self.shard_chars, self.depth - ))); - } - if self.name_encoding != LAYOUT_NAME_LOWER_HEX { - return Err(Error::Storage(format!( - "Chunk store names files with '{}', this build implements '{LAYOUT_NAME_LOWER_HEX}'", - self.name_encoding - ))); - } - Ok(()) - } -} - -/// Configuration for [`FileStore`]. -#[derive(Debug, Clone)] -pub struct FileStoreConfig { - /// Node root directory. The store lives at `{root_dir}/chunks/`. - pub root_dir: PathBuf, - /// Verify `BLAKE3(content) == address` on read. - pub verify_on_read: bool, - /// Free bytes to keep on the storage partition. Writes are refused below this. - pub disk_reserve: u64, -} - -/// Outcome of a single write attempt, used to keep the duplicate accounting honest. -enum PutOutcome { - /// The chunk was newly published. - New, - /// The chunk was already on disk. - Duplicate, -} - -/// Snapshot of free space, plus what has been written since it was taken. -#[derive(Debug)] -struct CapacitySnapshot { - /// When `available` was measured. `None` means never. - measured_at: Option, - /// Free bytes reported by the filesystem at `measured_at`. - available: u64, - /// Bytes published since `measured_at`, charged against `available`. - /// - /// Cleared by a fresh measurement, which already accounts for them. - written_since: u64, - /// Bytes reserved by writes that have not landed yet. - /// - /// Deliberately **not** cleared by a measurement: a `statvfs` taken while writes are - /// in flight reports space those writes are about to consume, so forgetting their - /// reservations at that moment would hand the same bytes out twice. That is precisely - /// the over-admission the reservation exists to prevent. - in_flight: u64, -} - -/// Size-aware free-space predicate with a short-lived cache. -/// -/// Free bytes alone stopped being a sufficient answer the moment chunks became files: -/// a caller wants to know whether *this* write fits, not whether the disk is non-empty. -/// The cache keeps the common case at one `statvfs` per interval while staying correct -/// under a burst, because bytes written since the measurement are charged against it. -#[derive(Debug)] -struct CapacityGuard { - /// Directory whose partition is measured. - dir: PathBuf, - /// Free bytes to keep unused. - reserve: u64, - /// The cached measurement. - snapshot: parking_lot::Mutex, -} - -impl CapacitySnapshot { - /// Free bytes, less everything written or promised since the measurement. - fn free_estimate(&self) -> u64 { - self.available - .saturating_sub(self.written_since) - .saturating_sub(self.in_flight) - } -} - -impl CapacityGuard { - /// Create a guard over the partition hosting `dir`. - fn new(dir: PathBuf, reserve: u64) -> Self { - Self { - dir, - reserve, - snapshot: parking_lot::Mutex::new(CapacitySnapshot { - measured_at: None, - available: 0, - written_since: 0, - in_flight: 0, - }), - } - } - - /// Bytes actually consumed on disk by a payload of `len` bytes. - fn charge(len: u64) -> u64 { - // Round the payload up to the allocation unit, then add one unit for the - // directory entry and inode. - len.div_ceil(ALLOC_UNIT) - .saturating_mul(ALLOC_UNIT) - .saturating_add(ALLOC_UNIT) - } - - /// Free bytes right now, or `None` if the question could not be answered. - /// - /// Deliberately separate from [`Self::measure`], which folds a failure into an error - /// the caller cannot tell from "below the reserve". - fn measure_available(&self) -> Option { - let mut snapshot = self.snapshot.lock(); - match self.measure(&mut snapshot) { - Ok(()) => Some(snapshot.free_estimate()), - Err(_) => None, - } - } - - /// Query the filesystem and refresh the snapshot. - fn measure(&self, snapshot: &mut CapacitySnapshot) -> Result<()> { - let available = fs2::available_space(&self.dir) - .map_err(|e| Error::Storage(format!("Failed to query available disk space: {e}")))?; - snapshot.available = available; - // Reservations survive: their bytes are not on the platter yet, so the fresh - // measurement does not include them. - snapshot.written_since = 0; - snapshot.measured_at = Some(Instant::now()); - Ok(()) - } - - /// Test `needed` against the snapshot, refreshing it if it is stale or short. - /// - /// Only *passing* results are cached, so a low-space condition is rechecked on every - /// call and freed space is noticed promptly. - fn admit(&self, snapshot: &mut CapacitySnapshot, needed: u64) -> Result<()> { - let want = self.reserve.saturating_add(Self::charge(needed)); - - let cache_fresh = snapshot - .measured_at - .is_some_and(|t| t.elapsed().as_secs() < DISK_CHECK_INTERVAL_SECS); - if cache_fresh && snapshot.free_estimate() >= want { - return Ok(()); - } - - self.measure(snapshot)?; - if snapshot.free_estimate() < want { - // Do not cache a failing result: `measured_at` is left set so the next call - // still re-measures, because the branch above only short-circuits a pass. - return Err(Error::Storage(format!( - "Insufficient disk space: {:.2} GiB available, {:.2} GiB reserve required. \ - Free disk space or increase the partition to continue storing chunks.", - bytes_to_gib(snapshot.free_estimate()), - bytes_to_gib(self.reserve), - ))); - } - Ok(()) - } - - /// Drop the cached measurement so the next question hits the filesystem. - fn invalidate(&self) { - let mut snapshot = self.snapshot.lock(); - snapshot.measured_at = None; - snapshot.written_since = 0; - } - - /// Return `Ok(())` if a write of `needed` bytes would fit. Charges nothing. - fn check(&self, needed: u64) -> Result<()> { - let mut snapshot = self.snapshot.lock(); - self.admit(&mut snapshot, needed) - } - - /// Admit a write of `needed` bytes and charge it in the same critical section. - /// - /// Checking and charging separately is the bug this exists to prevent: dozens of - /// protocol handlers can each pass against the same cached measurement before any of - /// them has written a byte, and collectively cross the reserve. - /// - /// The returned [`Reservation`] settles itself when dropped, so a caller whose future - /// is dropped mid-write cannot strand it. Nothing else ever decrements the in-flight - /// count, so a stranded reservation would be permanent, and enough of them would make - /// an empty disk look full until the process restarted. - fn reserve(self: &Arc, needed: u64) -> Result { - { - let mut snapshot = self.snapshot.lock(); - self.admit(&mut snapshot, needed)?; - snapshot.in_flight = snapshot.in_flight.saturating_add(Self::charge(needed)); - } - Ok(Reservation { - capacity: Arc::clone(self), - bytes: needed, - settled: false, - }) - } - - /// Give back a reservation whose write did not happen. - fn release(&self, needed: u64) { - let mut snapshot = self.snapshot.lock(); - snapshot.in_flight = snapshot.in_flight.saturating_sub(Self::charge(needed)); - } - - /// Turn a reservation into bytes that are now on disk. - fn commit_reservation(&self, needed: u64) { - let charge = Self::charge(needed); - let mut snapshot = self.snapshot.lock(); - snapshot.in_flight = snapshot.in_flight.saturating_sub(charge); - snapshot.written_since = snapshot.written_since.saturating_add(charge); - } - - /// Credit a completed delete back to the cached measurement. - fn record_removed(&self, len: u64) { - let mut snapshot = self.snapshot.lock(); - snapshot.written_since = snapshot.written_since.saturating_sub(Self::charge(len)); - } -} - -/// A charged, unsettled write. -/// -/// Held by whatever is actually doing the write, so the charge is released even if the -/// caller's future is dropped and only the blocking closure survives. -struct Reservation { - /// The guard this was taken from. - capacity: Arc, - /// Payload size, before rounding. - bytes: u64, - /// Whether it has already been accounted for. - settled: bool, -} - -impl Reservation { - /// The write landed: move the charge from in-flight to written. - fn commit(mut self) { - self.capacity.commit_reservation(self.bytes); - self.settled = true; - } -} - -impl Drop for Reservation { - fn drop(&mut self) { - if !self.settled { - self.capacity.release(self.bytes); - } - } -} - -/// Environment variable naming a failpoint: stop after the temp file, before the rename. -#[cfg(any(test, feature = "test-utils"))] -pub const HALT_BEFORE_PUBLISH: &str = "ANT_HALT_BEFORE_PUBLISH"; - -/// Environment variable naming a failpoint: stop once the legacy environment is renamed -/// aside and marked retired, before any of it is deleted. -/// -/// The most destructive window in the migration. What a start that finds a marked -/// directory must do is finish the deletion, never reopen it, because the node has already -/// told the network it holds those chunks from the file store. -#[cfg(any(test, feature = "test-utils"))] -pub const HALT_AFTER_RETIRE_MARK: &str = "ANT_HALT_AFTER_RETIRE_MARK"; - -/// Park forever at a named failpoint, once a marker says the process has reached it. -/// -/// For crash tests, which need a process to die *inside* an operation rather than at -/// whatever point a sleep in another process happened to land. The variable holds a path: -/// this writes it, so the parent knows the child is exactly here, and then waits to be -/// killed. -/// -/// Costs one environment read per write when the feature is compiled in, and the feature -/// is not in a release build. -#[cfg(any(test, feature = "test-utils"))] -pub(crate) fn halt_here_if_asked(variable: &str, reached: &Path) { - let Ok(marker) = std::env::var(variable) else { - return; - }; - // Let the first few through. A test that stops the very first write leaves a store - // with nothing successfully in it, and an assertion over what it holds then passes by - // iterating nothing. Letting some land first means the crash happens to a store that - // has real chunks in it, which is the situation worth checking. - let skip: u64 = std::env::var(HALT_AFTER) - .ok() - .and_then(|raw| raw.parse().ok()) - .unwrap_or(0); - if HALTS_SEEN.fetch_add(1, std::sync::atomic::Ordering::AcqRel) < skip { - return; - } - if let Err(e) = std::fs::write(&marker, reached.as_os_str().as_encoded_bytes()) { - // The parent waits for this file. Saying so on the way past is the difference - // between a test that fails and one that hangs until the job times out. - eprintln!("failpoint could not write its marker {marker}: {e}"); - return; - } - loop { - std::thread::sleep(Duration::from_secs(3600)); - } -} - -/// How many writes to let through before the failpoint fires. -#[cfg(any(test, feature = "test-utils"))] -pub const HALT_AFTER: &str = "ANT_HALT_AFTER"; - -/// How many times the failpoint has been reached in this process. -#[cfg(any(test, feature = "test-utils"))] -static HALTS_SEEN: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); - -/// Clears a write's registration when the work finishes, however it finishes. -/// -/// Held by the blocking closure rather than by the caller, so a dropped future cannot -/// leave an entry behind, and a panic in the work cannot either. -struct WriteInFlight { - writing: Arc>>, - finished: Arc, - address: XorName, -} - -impl Drop for WriteInFlight { - fn drop(&mut self) { - let was_last = { - let mut writing = self.writing.lock(); - match writing.get_mut(&self.address) { - Some(count) if *count > 1 => { - *count -= 1; - false - } - _ => { - writing.remove(&self.address); - true - } - } - }; - // Only when this was the last one. Waking a waiter while another write for the - // same key is still queued is exactly what the count exists to prevent. - if was_last { - self.finished.notify_waiters(); - } - } -} - -/// What is behind a chunk's name on disk. -/// -/// Four answers, not two, because "could not read it" must never be treated as "wrong": -/// replacing a chunk is destructive, and off Unix it truncates the file in place, so a -/// transient fault would turn a healthy sole copy into an empty one. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum StoredBytes { - /// The bytes are there and hash to the name. - Good, - /// The bytes are there and do not. - Wrong, - /// There is nothing behind the name. - Absent, - /// The question could not be answered this time. - Unreadable, -} - -/// Convert a byte count to GiB for human-readable log messages. -#[allow(clippy::cast_precision_loss)] // display only — sub-byte precision is irrelevant -fn bytes_to_gib(bytes: u64) -> f64 { - bytes as f64 / (1024.0 * 1024.0 * 1024.0) -} - -/// Content-addressed store holding one immutable file per chunk. -/// -/// The filesystem is the sole authority. The in-memory index is a cache of what the -/// directory tree already contains, rebuilt from directory entries at every open, and -/// every mutation of it mirrors a filesystem operation that has *already* completed. -/// Bitcask's issue #114 is the cautionary tale for the opposite order: an index that is -/// rebuilt at startup and then mutated in anticipation drifts, and the drift is silent. -#[derive(Debug)] -pub struct FileStore { - /// Store configuration. - config: FileStoreConfig, - /// `{root_dir}/chunks`. - chunks_dir: PathBuf, - /// Every address whose file is published, in ascending order. - /// - /// `BTreeSet` rather than a hash set because `all_keys()` must be sorted (the - /// commitment builder truncates with `take(cap)` *before* the Merkle tree sorts, so - /// an unstable order would make the node's published commitment depend on iteration - /// luck), and because it never spikes memory while growing. - index: Arc>>, - /// One mutex per shard, serialising writers of the same address. - /// - /// LMDB gave exactly-once `put` semantics for free: the duplicate test happened - /// inside the write transaction. Two threads publishing the same address here would - /// otherwise both see an absent file, both rename, and both report "newly stored", - /// double-counting the chunk. The lane is indexed by the address's LAST byte for the - /// same reason the shard is: a node's keys share their leading bytes, so lanes keyed - /// on the first byte would all collapse into one. - write_lanes: Arc>>, - /// Operation counters, same shape as the LMDB store reported. - stats: parking_lot::RwLock, - /// Which of the 256 shard directories are known to exist, so a steady-state write - /// does not pay a `create_dir_all` syscall. - shards_present: Arc>, - /// Indexed chunks this store currently cannot read. - /// - /// Held back from everything the node says it has, while the files themselves are - /// left alone. See [`Self::mark_suspect`]. - suspect: Arc>>, - /// Indexed chunks a read has proven do not match their name. - /// - /// Separate from the above because they clear differently. Not being able to read a - /// file is a question a later read answers; bytes that are wrong stay wrong however - /// often they are read, and only a repair or a removal settles it. A raw read that - /// does not hash anything must not take a chunk out of this set. - known_wrong: Arc>>, - /// Addresses this store is part-way through writing. - /// - /// Every mutation registers here before it spawns its blocking work and clears the - /// entry *inside* that work, so a caller whose future is dropped cannot skip the - /// clearing while the write itself goes on to land. That is the difference that - /// matters: the blocking half is not cancelled with the future, so anything the - /// future was going to do afterwards is not a record of what happened. - /// - /// It lets a delete queue behind the exact write it would otherwise race, rather than - /// behind every write this store has in flight. - /// - /// Counted, not a set. Cancellation can release the facade's key lane while the - /// blocking half survives, so a second write for the same key can start behind the - /// first. With one entry between them, whichever finished first would remove it and a - /// waiter would be told the key is free while the other was still queued. - writing: Arc>>, - /// Woken when [`Self::writing`] loses its last entry for a key. - write_finished: Arc, - /// Bumped whenever a chunk stops being servable. - /// - /// The pre-retirement pass reads every chunk, and its result is reused for a while - /// rather than re-read on every tick. This is how the caller can tell that the store - /// has not changed underneath that result: a proof carries the value it saw, and a - /// file that has since gone or stopped being readable makes it stale. - health: Arc, - /// Size-aware free-space predicate. - capacity: Arc, - /// Monotonic counter that makes temp filenames unique within this store. - temp_seq: AtomicU64, - /// Random per-instance discriminator for temp filenames. - nonce: u32, - /// Held for the store's lifetime. Startup fails without it. - /// - /// Shared rather than owned so the blocking work that depends on it can hold a lease - /// of its own: that work outlives the future that spawned it, and a cancelled caller - /// releasing the lock would leave it writing into a directory another process had - /// just been let into. - lock: Arc, - /// Tracks every blocking task, so [`FileStore::wait_idle`] can wait for writes that - /// outlived their awaiting future. - blocking_tracker: TaskTracker, - /// Test-only gate read-acquired at the top of the put blocking closure. - /// - /// Tests hold the write half to park an in-flight write on the blocking pool, which - /// is the shape a `select!` losing to a shutdown token leaves behind. - #[cfg(any(test, feature = "test-utils"))] - test_put_gate: Arc>, -} - -impl FileStore { - /// Open (or create) the store at `{root_dir}/chunks/`. - /// - /// Sweeps orphaned temp files, then rebuilds the index from directory entries. - /// The scan reads names only: it never `stat`s an entry and never reads a chunk. - /// - /// # Errors - /// - /// Returns [`Error::Storage`] if the directory cannot be created, the layout marker - /// is unreadable or describes a layout this build does not implement, or the scan - /// fails. - pub async fn new(config: FileStoreConfig) -> Result { - let chunks_dir = config.root_dir.join(CHUNKS_DIR_NAME); - std::fs::create_dir_all(&chunks_dir).map_err(|e| { - Error::Storage(format!( - "Failed to create chunk store directory {}: {e}", - chunks_dir.display() - )) - })?; - - check_path_budget(&chunks_dir); - - let layout = read_or_write_layout(&chunks_dir)?; - layout.check_supported()?; - - // Startup fails without it, so from here this process is the only one using this - // directory and an interrupted write can only be its own. - let lock = acquire_store_lock(&chunks_dir)?; - - let scan_dir = chunks_dir.clone(); - // The scan holds the lease itself. It sweeps interrupted writes on the strength of - // being alone here, and it runs on a thread that outlives this future: a - // cancelled startup that released the lock would leave it sweeping a directory - // another process had just been let into. - let scan_lease = Arc::clone(&lock); - // The node root as well as the chunk tree. The scan sweeps interrupted writes - // under `chunks/`, which covers the layout marker's temporary because that lives - // there; the migration marker's lives in the root, where nothing looked. - let root = config.root_dir.clone(); - let scan = spawn_blocking(move || { - let _lease = scan_lease; - sweep_marker_temps(&root); - scan_store(&scan_dir) - }) - .await - .map_err(|e| Error::Storage(format!("Chunk store scan task failed: {e}")))??; - - let ScanResult { - keys, - shards_present, - swept_temps, - skipped, - } = scan; - - let key_count = keys.len(); - // Build from a sorted vector: bulk-building packs every B-tree node to its - // capacity, where repeated `insert` converges on ~68% fill for the same keys. - let index: BTreeSet = keys.into_iter().collect(); - - if swept_temps > 0 { - info!("Chunk store: removed {swept_temps} orphaned temporary file(s) from interrupted writes"); - } - if skipped > 0 { - warn!("Chunk store: ignored {skipped} directory entr(ies) that are not chunk files"); - } - info!( - "Chunk store open at {} ({key_count} chunks)", - chunks_dir.display() - ); - - let capacity = Arc::new(CapacityGuard::new(chunks_dir.clone(), config.disk_reserve)); - - Ok(Self { - config, - chunks_dir, - index: Arc::new(parking_lot::RwLock::new(index)), - write_lanes: Arc::new( - std::iter::repeat_with(|| parking_lot::Mutex::new(())) - .take(SHARD_COUNT) - .collect(), - ), - stats: parking_lot::RwLock::new(StorageStats::default()), - shards_present: Arc::new(parking_lot::Mutex::new(shards_present)), - suspect: Arc::new(parking_lot::RwLock::new(HashSet::new())), - known_wrong: Arc::new(parking_lot::RwLock::new(HashSet::new())), - writing: Arc::new(parking_lot::Mutex::new(HashMap::new())), - write_finished: Arc::new(tokio::sync::Notify::new()), - health: Arc::new(std::sync::atomic::AtomicU64::new(0)), - capacity, - temp_seq: AtomicU64::new(0), - nonce: rand::random(), - lock, - blocking_tracker: TaskTracker::new(), - #[cfg(any(test, feature = "test-utils"))] - test_put_gate: Arc::new(parking_lot::RwLock::new(())), - }) - } - - /// Store a chunk. - /// - /// On Unix, publishing is a rename within the destination directory, so the final name - /// can never appear on partial content: the name *is* the hash, and the content is - /// fully written and flushed before the name exists. Off Unix there is no rename, for - /// the reason `publish_in_place` gives (it is compiled only on those platforms, so this - /// is not a link), and a partial file can wear a real name; that - /// is why a duplicate is read and compared rather than trusted. - /// - /// # Returns - /// - /// `true` if the chunk was newly stored, `false` if it was already present. - /// - /// # Errors - /// - /// Returns [`Error::Storage`] if the content does not hash to `address`, the disk - /// is too full, or the write fails. - pub async fn put(&self, address: &XorName, content: &[u8]) -> Result { - let computed = crate::client::compute_address(content); - if computed != *address { - return Err(Error::Storage(format!( - "Content address mismatch: expected {}, computed {}", - hex::encode(address), - hex::encode(computed) - ))); - } - // The read path refuses anything over the ceiling, so writing one would create a - // file the store could never read back and could never repair. - if content.len() > MAX_CHUNK_SIZE { - return Err(Error::Storage(format!( - "Chunk {} is {} bytes, over the {MAX_CHUNK_SIZE} byte maximum", - hex::encode(address), - content.len() - ))); - } - - // An indexed name is not proof of the bytes under it. The index is built from - // names, by the startup scan and by a completed publish, and a name can outlive - // what it points at: off Unix a chunk is created under its final name before its - // bytes are written, so a crash leaves a short file wearing a real name, and rot - // leaves a full-length one. Answering "already have it" to the copy that would fix - // either is how a node discards its own repair and is never offered another. - // - // So the bytes decide. Checked before the reservation below, so re-storing a chunk - // this node already holds stays a no-op on a full disk. - if self.index.read().contains(address) { - if let Some(answer) = self.settle_indexed_duplicate(address, content).await { - return answer; - } - } - - let len = content.len() as u64; - // Reserved after the duplicate test so re-storing an existing chunk stays a - // harmless no-op on a full disk, matching the LMDB store's ordering. - let reservation = self.capacity.reserve(len)?; - - let shard = self.chunks_dir.join(shard_name(address)); - let final_path = shard.join(hex::encode(address)); - let temp_path = shard.join(self.next_temp_name()); - let payload = content.to_vec(); - let lanes = Arc::clone(&self.write_lanes); - let index = Arc::clone(&self.index); - let shards_present = Arc::clone(&self.shards_present); - let chunks_dir = self.chunks_dir.clone(); - let lane = shard_index(address); - let key = *address; - #[cfg(any(test, feature = "test-utils"))] - let test_put_gate = Arc::clone(&self.test_put_gate); - // Registered before the work is spawned and cleared by the work itself, so a - // caller that goes away cannot leave a delete free to race this publish. - let in_flight = self.begin_write(address); - // And the lease, for the same reason the scan holds it: this thread writes into a - // directory whose exclusivity the lock is what establishes, and it can outlive - // the last owner of the store. - let lease = Arc::clone(&self.lock); - let known_wrong = Arc::clone(&self.known_wrong); - let suspect = Arc::clone(&self.suspect); - - let outcome = self - .blocking_tracker - .spawn_blocking(move || -> Result { - let _in_flight = in_flight; - let _lease = lease; - // Test-only: parks here while a test holds the write half. - #[cfg(any(test, feature = "test-utils"))] - let _test_put_gate = test_put_gate.read(); - let _lane = lanes.get(lane).map(parking_lot::Mutex::lock); - // `mkdir` plus a directory flush are syscalls, so they belong here and - // not on a runtime worker. - ensure_shard_dir(&chunks_dir, &shard, lane, &shards_present)?; - let outcome = match publish(&temp_path, &final_path, &payload, &shard) { - Ok(outcome) => outcome, - Err(PublishFailed { error, left_behind }) => { - // A publish that failed can still have left the bytes there: off - // Unix the chunk is created under its final name, and if the write - // or the flush then fails, the cleanup that removes it can fail - // too. Releasing the reservation would hand back a charge for a - // file that is on the disk. - // - // The publish says so rather than this deciding from a later - // `is_file`. Asking the filesystem afterwards infers ownership from - // a name being occupied, which is true under the store lock and the - // shard lane and not true against anything out of band, and this - // file spends a lot of its length arguing that a name is not - // evidence. A bit set by the code that created the file is. - if left_behind { - reservation.commit(); - } - return Err(error); - } - }; - // Placed, not yet durable. A failure from here on leaves the bytes on the - // disk: the chunk is rightly not reported as stored, because a copy that is - // not durable must not authorise deleting another, but the space is spent - // all the same. Dropping the reservation would hand that charge back and - // admit the next write against room that is already gone. - // - // Only for a chunk this call published. `Duplicate` means the file was - // already there and was charged by whoever wrote it, so charging it again - // here would count one file twice and shrink the store's idea of its own - // disk on every retry. - if let Err(e) = flush_publication(&final_path, &shard) { - if matches!(outcome, PutOutcome::New) { - reservation.commit(); - } - return Err(e); - } - // Index inside the lane, and only after the rename has returned. A - // concurrent delete of the same address therefore cannot interleave - // between publishing the file and admitting the key. - // - // Only for a chunk this call actually published. `Duplicate` says a file - // already wears the name, and a name is not evidence about the bytes under - // it: the four-way answer that decides whether they are good, wrong, absent - // or unreadable runs after the await below, and a caller whose future is - // dropped never reaches it. Admitting the key here would leave the node - // claiming, advertising and committing to bytes nothing has read, with no - // suspect or known-wrong mark to hold it back, and the sharpest case is a - // name the startup scan deliberately refused because what wears it is a - // fifo, a socket or a directory. The duplicate arm admits the key itself, - // once a read has proven the bytes. - if matches!(outcome, PutOutcome::New) { - index.write().insert(key); - // With the marks that would otherwise hold the key back. These bytes - // were hashed against their own name on the way in, so an older - // instance proven wrong or merely unreadable has just been replaced by - // a good one. Cleared here rather than after the await for the same - // reason the insert is here: a cancelled caller would leave the key - // indexed and suppressed at once, so a chunk this node really does hold - // would stay hidden from `exists` and `all_keys` until some later read - // happened to settle it. - known_wrong.write().remove(&key); - suspect.write().remove(&key); - // Settled here, inside the work, so a dropped awaiter cannot strand it. - reservation.commit(); - } - Ok(outcome) - }) - .await - .map_err(|e| Error::Storage(format!("Chunk store put task failed: {e}")))??; - - match outcome { - PutOutcome::Duplicate => self.settle_duplicate(address, content).await, - PutOutcome::New => { - // Freshly published bytes that were checked against their own name on the - // way in. The marks were already cleared inside the work, where a dropped - // caller cannot skip them; what is left here is only what a caller who is - // still waiting should see. - let mut stats = self.stats.write(); - stats.chunks_stored = stats.chunks_stored.saturating_add(1); - stats.bytes_stored = stats.bytes_stored.saturating_add(len); - drop(stats); - debug!("Stored chunk {} ({len} bytes)", hex::encode(address)); - Ok(true) - } - } - } - - /// Decide what a name that was already taken actually means. - /// - /// Split out of [`Self::put`] because it is a different question. `put` puts bytes on - /// a disk; this reads bytes back to find out whether the ones already there are the - /// ones the caller is offering, which is the only thing that makes a duplicate safe to - /// report as stored. - /// - /// # Errors - /// - /// Returns [`Error::Storage`] when the existing file is absent or could not be read, - /// both of which mean this node must not report the chunk as held. - async fn settle_duplicate(&self, address: &XorName, content: &[u8]) -> Result { - // The file was already on disk, and its name is not evidence its contents - // are right. The startup scan indexes by name without reading anything, - // and on Windows a crash mid-write leaves a partial file under a real - // chunk name. Trusting the name here would acknowledge a chunk that was - // never stored, and then discard the good copy arriving to repair it. - // Every answer handled, because three of the four must not report the - // chunk as stored. A caller that hears success acts on it: a client drops - // its own copy, replication marks the key held, and the copier takes it - // out of the legacy-only set. - match self.stored_bytes_match(address).await { - StoredBytes::Good => { - // Admitted here, which is the first moment the bytes behind the - // name have been read and shown to hash to it. Idempotent: the - // ordinary case is a key the startup scan already indexed. - self.index.write().insert(*address); - { - let mut stats = self.stats.write(); - stats.duplicates = stats.duplicates.saturating_add(1); - } - Ok(false) - } - StoredBytes::Wrong => { - warn!( - "Chunk {} was already on disk but its contents are wrong; \ - replacing it with the copy just offered", - hex::encode(address) - ); - self.repair(address, content).await.map(|()| true) - } - // The name was taken a moment ago and is not now, or was never a - // readable chunk file. Either way nothing holds these bytes, so say so - // rather than reporting a chunk that is not there. - StoredBytes::Absent => Err(Error::Storage(format!( - "Chunk {} was reported already on disk but nothing is there. Not \ - reporting it as stored.", - hex::encode(address) - ))), - // Replacing on an unanswered question would destroy a healthy copy, - // and reporting success would discard the offered one. The index entry - // stays: the file is still there, and dropping the entry would leave - // the chunk in neither this store's view nor the legacy one, which is - // what retirement destroys. Removing an entry is the quarantine path's - // job, and it removes the file with it, after a read that succeeded - // and proved the bytes wrong. - StoredBytes::Unreadable => Err(Error::Storage(format!( - "Chunk {} is on disk but could not be read to check it. Not \ - replacing it, and not reporting it as stored.", - hex::encode(address) - ))), - } - } - - /// Flush every directory a chunk can live in, so the names in them are durable. - /// - /// Byte integrity is not the whole of what the pre-retirement proof has to establish. - /// A chunk whose contents are on the platter but whose *name* is not is still lost to - /// a power loss, and a publish whose rename landed and whose directory flush failed - /// leaves exactly that: the next attempt sees the name, the next verification reads - /// the right bytes, and nothing goes back to retry the flush. So the proof flushes - /// them itself rather than trusting that each publish did. - /// - /// Cheap: at most 257 directory flushes for a store of any size, and nothing off Unix, - /// where directories cannot be flushed and the retirement marker covers the same - /// ground instead. - /// - /// # Errors - /// - /// Returns [`Error::Storage`] on the first directory that cannot be flushed. The - /// caller must treat that as a proof it did not get. - pub fn flush_namespace(&self) -> Result<()> { - fsync_dir(&self.chunks_dir).map_err(|e| { - Error::Storage(format!( - "Could not flush {}: {e}", - self.chunks_dir.display() - )) - })?; - let present = *self.shards_present.lock(); - for (shard, _) in present.iter().enumerate().filter(|(_, here)| **here) { - let dir = self.chunks_dir.join(format!("{shard:02x}")); - fsync_dir(&dir) - .map_err(|e| Error::Storage(format!("Could not flush {}: {e}", dir.display())))?; - } - Ok(()) - } - - /// Decide what to do about a write of a chunk the index already names. - /// - /// `None` means the index was wrong and there is nothing on disk, so the caller - /// publishes it as new. Everything else is the answer. - async fn settle_indexed_duplicate( - &self, - address: &XorName, - content: &[u8], - ) -> Option> { - match self.stored_bytes_match(address).await { - StoredBytes::Good => { - trace!("Chunk {} already exists", hex::encode(address)); - { - let mut stats = self.stats.write(); - stats.duplicates = stats.duplicates.saturating_add(1); - } - Some(Ok(false)) - } - StoredBytes::Wrong => { - warn!( - "Chunk {} is indexed but its bytes are wrong; replacing it with the \ - copy just offered", - hex::encode(address) - ); - Some(self.repair(address, content).await.map(|()| true)) - } - // Indexed but gone: publish it fresh rather than replacing something that is - // not there. - StoredBytes::Absent => None, - // Unanswerable this time. Do not touch what is there, and do not tell the - // caller the chunk is safely stored either: a client would take that as an - // acknowledgement and drop the only other copy. The index entry stays, for - // the reason given on the same case after publication. - StoredBytes::Unreadable => Some(Err(Error::Storage(format!( - "Chunk {} is indexed but could not be read to check it. Not replacing it, \ - and not reporting it as stored.", - hex::encode(address) - )))), - } - } - - /// Drop an address from the index without touching the file. Tests only. - /// - /// Stands in for whatever leaves a key indexed nowhere: a quarantine, a publish that - /// failed after the file went, an operator with a shell. - #[cfg(test)] - pub(crate) fn forget_for_test(&self, address: &XorName) { - self.index.write().remove(address); - } - - /// The size of the file behind `address`, if there is one. - /// - /// One `metadata` call, no read. Used where an indexed name has to be checked against - /// what a caller is offering before that offer is turned away. - #[must_use] - pub fn stored_len(&self, address: &XorName) -> Option { - std::fs::metadata(self.chunk_path(address)) - .ok() - .filter(std::fs::Metadata::is_file) - .and_then(|m| usize::try_from(m.len()).ok()) - } - - /// Whether the file already stored under `address` really hashes to it. - async fn stored_bytes_match(&self, address: &XorName) -> StoredBytes { - match self.get_raw(address).await { - Ok(Some(bytes)) if crate::client::compute_address(&bytes) == *address => { - // A read that hashed. It settles both questions. - self.clear_suspect(address); - self.clear_known_wrong(address); - StoredBytes::Good - } - Ok(Some(_)) => { - self.mark_known_wrong(address); - StoredBytes::Wrong - } - Ok(None) => StoredBytes::Absent, - // NOT the same as wrong. A file that could not be read this once may be - // perfectly good, and off Unix replacing it means opening it with `truncate`, - // which would destroy a healthy sole copy on the strength of a transient - // fault. Say so and let the caller leave it alone. - Err(e) => { - debug!("Could not read {} to check it: {e}", hex::encode(address)); - self.mark_suspect(address); - StoredBytes::Unreadable - } - } - } - - /// Replace the file behind an address with known-good bytes, atomically. - /// - /// Unlike [`Self::put`], this deliberately publishes **over** an existing name. It - /// exists for one caller: repairing a file whose bytes no longer hash to their own - /// name, from a copy held elsewhere, before that copy is destroyed. Doing it as - /// delete-then-put would leave a window where the only remaining copy is the one - /// about to be deleted, and any failure in that window is unrecoverable. - /// - /// # Errors - /// - /// Returns [`Error::Storage`] if `content` does not hash to `address`, or the write - /// fails. The old file is left untouched on every error path. - pub async fn repair(&self, address: &XorName, content: &[u8]) -> Result<()> { - // The same ceiling `put` enforces. Without it a repair can install bytes the read - // path will refuse for ever, which is a chunk that verifies as present and can - // never be served. - if content.len() > MAX_CHUNK_SIZE { - return Err(Error::Storage(format!( - "Refusing to repair {} with {} bytes, over the {MAX_CHUNK_SIZE} byte \ - maximum", - hex::encode(address), - content.len() - ))); - } - let computed = crate::client::compute_address(content); - if computed != *address { - return Err(Error::Storage(format!( - "Refusing to repair {} with content that hashes to {}", - hex::encode(address), - hex::encode(computed) - ))); - } - // The replacement exists alongside the original until the rename, so the room for - // it has to be there first. Reserved rather than merely checked: a plain check - // passes against a cached measurement, so concurrent repairs and PUTs can each be - // admitted against the same headroom and cross the reserve together. - // Moved into the work below, so it is released when the write finishes rather - // than when its caller stops waiting. A caller that goes away otherwise frees - // room that the detached write is still about to consume. - let reservation = self.capacity.reserve(content.len() as u64)?; - - let shard = self.chunks_dir.join(shard_name(address)); - let final_path = shard.join(hex::encode(address)); - let temp_path = shard.join(self.next_temp_name()); - let payload = content.to_vec(); - let lanes = Arc::clone(&self.write_lanes); - let index = Arc::clone(&self.index); - let shards_present = Arc::clone(&self.shards_present); - let chunks_dir = self.chunks_dir.clone(); - let lane = shard_index(address); - let key = *address; - let in_flight = self.begin_write(address); - let lease = Arc::clone(&self.lock); - let capacity = Arc::clone(&self.capacity); - let suspect = Arc::clone(&self.suspect); - let known_wrong = Arc::clone(&self.known_wrong); - - self.blocking_tracker - .spawn_blocking(move || -> Result<()> { - let _in_flight = in_flight; - let _lease = lease; - let _reservation = reservation; - let _lane = lanes.get(lane).map(parking_lot::Mutex::lock); - ensure_shard_dir(&chunks_dir, &shard, lane, &shards_present)?; - write_and_replace(&temp_path, &final_path, &payload, &shard)?; - index.write().insert(key); - // Settled here rather than after the await. The replacement has landed - // and hashes to its own name, so nothing is wrong with this chunk any - // more; a caller that stopped waiting would otherwise leave a healthy - // file excluded from everything the node claims to hold, and the - // measurement believing the store is a chunk smaller than it is. - suspect.write().remove(&key); - known_wrong.write().remove(&key); - capacity.invalidate(); - Ok(()) - }) - .await - .map_err(|e| Error::Storage(format!("Chunk store repair task failed: {e}")))??; - - // Everything the success means was recorded by the work that succeeded: the - // reservation released, the marks cleared, the measurement thrown away. Released - // rather than committed because a repair is not a new chunk, and the measurement - // discarded rather than adjusted because the file it replaced may have been - // shorter, which is exactly the case a repair fixes. - debug!("Repaired chunk {}", hex::encode(address)); - Ok(()) - } - - /// Retrieve a chunk, verifying it against its address when configured to. - /// - /// # Errors - /// - /// Returns [`Error::Storage`] on an I/O failure, or when verification fails. A - /// chunk whose bytes do not hash to its name is removed and dropped from the index - /// before the error is returned, so it leaves `all_keys()` and ordinary replication - /// repairs it. - pub async fn get(&self, address: &XorName) -> Result>> { - let Some(content) = self.read_file(address).await? else { - trace!("Chunk {} not found", hex::encode(address)); - return Ok(None); - }; - - if self.config.verify_on_read { - let computed = crate::client::compute_address(&content); - if computed != *address { - { - let mut stats = self.stats.write(); - stats.verification_failures = stats.verification_failures.saturating_add(1); - } - warn!( - "Chunk verification failed: expected {}, computed {}", - hex::encode(address), - hex::encode(computed) - ); - // Said before it is acted on. Removing the file can fail or be cancelled, - // and a chunk proven wrong that goes on looking healthy is one the node - // keeps committing to and, worse, one a cached pre-retirement pass still - // covers: the legacy copy that would repair it gets deleted. - self.mark_known_wrong(address); - self.quarantine_corrupt(address).await; - return Err(Error::Storage(format!( - "Chunk verification failed for {}", - hex::encode(address) - ))); - } - } - - if self.config.verify_on_read { - // The bytes hashed to their name. Whatever this store thought was wrong with - // them is not wrong with them, and a mark that outlives the fault it - // describes means the node can serve a chunk it will not claim, commit or - // offer. - self.clear_known_wrong(address); - } - - let len = content.len() as u64; - { - let mut stats = self.stats.write(); - stats.chunks_retrieved = stats.chunks_retrieved.saturating_add(1); - stats.bytes_retrieved = stats.bytes_retrieved.saturating_add(len); - } - debug!("Retrieved chunk {} ({len} bytes)", hex::encode(address)); - Ok(Some(content)) - } - - /// Retrieve raw chunk bytes without content-address verification. - /// - /// # Errors - /// - /// Returns [`Error::Storage`] on an I/O failure. - pub async fn get_raw(&self, address: &XorName) -> Result>> { - self.read_file(address).await - } - - /// Check whether a chunk is stored. - /// - /// An in-memory lookup: no syscall, no I/O. - /// - /// # Errors - /// - /// Never fails. The signature keeps the shape the LMDB store had, because callers - /// treat the error as "assume absent". - pub fn exists(&self, address: &XorName) -> Result { - if self.is_unservable(address) { - return Ok(false); - } - Ok(self.is_indexed(address)) - } - - /// Is this chunk one the node must not answer for? - #[must_use] - fn is_unservable(&self, address: &XorName) -> bool { - self.suspect.read().contains(address) || self.known_wrong.read().contains(address) - } - - /// Is this chunk in the index, whether or not it can currently be read? - /// - /// The physical question, as against [`Self::exists`]'s question about what the node - /// is willing to claim. The migration must ask this one: a suspect chunk is still a - /// file this store has, and treating it as absent would put the key in the legacy-only - /// set, from where the union view advertises it again — a key the node claims through - /// one view and cannot serve through either. - #[must_use] - pub fn is_indexed(&self, address: &XorName) -> bool { - self.index.read().contains(address) - } - - /// Delete a chunk, returning whether it was present. - /// - /// `unlink` returns the blocks to the filesystem immediately. That is the whole - /// point of this store: no free list, no compaction, no free space required to - /// reclaim space. - /// - /// # Errors - /// - /// Returns [`Error::Storage`] if the file exists but cannot be removed. The index - /// keeps the key in that case, because the bytes are still on disk. - pub async fn delete(&self, address: &XorName) -> Result { - let path = self.chunk_path(address); - let lanes = Arc::clone(&self.write_lanes); - let index = Arc::clone(&self.index); - let lane = shard_index(address); - let key = *address; - // Carried into the closure for the reason `put`, `repair` and the startup scan - // carry it: this work outlives the future that started it, so a cancelled caller - // that drops the last `FileStore` would otherwise release the directory to another - // process while an unlink is still queued against it. Deleting is the operation - // where that matters most. - let lease = Arc::clone(&self.lock); - - let (existed, freed) = self - .blocking_tracker - .spawn_blocking(move || -> Result<(bool, u64)> { - let _lease = lease; - let _lane = lanes.get(lane).map(parking_lot::Mutex::lock); - let len = std::fs::metadata(&path).map_or(0, |m| m.len()); - let removed = match std::fs::remove_file(&path) { - Ok(()) => { - // Without this a crash can resurrect the entry on ext4, XFS, - // btrfs and APFS: the unlink is in the page cache, the directory - // is not. - if let Some(shard) = path.parent() { - fsync_dir_best_effort(shard); - } - true - } - // Already gone: the index was stale. Still a successful delete as - // far as the caller is concerned. - Err(e) if e.kind() == ErrorKind::NotFound => false, - Err(e) => { - return Err(Error::Storage(format!( - "Failed to delete chunk file {}: {e}", - path.display() - ))) - } - }; - // Index only after the filesystem operation has succeeded. On the error - // path above the entry stays, because the bytes are still on disk. - let was_indexed = index.write().remove(&key); - Ok((removed || was_indexed, if removed { len } else { 0 })) - }) - .await - .map_err(|e| Error::Storage(format!("Chunk store delete task failed: {e}")))??; - - if freed > 0 { - self.capacity.record_removed(freed); - debug!("Deleted chunk {}", hex::encode(address)); - } - Ok(existed) - } - - /// Return every stored key, in ascending order. - /// - /// The order is a correctness requirement, not a convenience: the commitment - /// builder truncates the responsible subset with `take(cap)` before the Merkle tree - /// sorts it, so an unstable order would make the node's published commitment depend - /// on iteration luck. - /// - /// # Errors - /// - /// Never fails. The signature matches the LMDB store's. - // Async without awaiting anything, deliberately: the whole point of this store is - // that the key set is already in memory. Callers are spread across the replication - // engine and cannot all be de-async'd in this change. - // - // Two lint names because they were renamed between toolchains, and `unknown_lints` - // so whichever one the compiler in use has never heard of stays quiet. - #[allow(unknown_lints)] - #[allow(clippy::unused_async, clippy::unused_async_trait_impl)] - pub async fn all_keys(&self) -> Result> { - // Copied out first so neither lock is held while the other is taken, and so the - // usual case, where nothing is suspect, costs one clone of an empty set. - let mut unservable: HashSet = self.suspect.read().clone(); - unservable.extend(self.known_wrong.read().iter().copied()); - let keys = self.index.read().clone(); - if unservable.is_empty() { - return Ok(keys.into_iter().collect()); - } - Ok(keys - .into_iter() - .filter(|key| !unservable.contains(key)) - .collect()) - } - - /// Stop answering for a chunk this store could not read. - /// - /// The file stays. It may be perfectly good and unreadable only for the moment, and - /// deleting it, or dropping it from the index, is how a chunk ends up in neither this - /// store's view nor the legacy one, which is what retirement destroys. - /// - /// What does change is what the node says about it. A chunk it cannot read is one it - /// cannot serve, and claiming it anyway puts the key in signed commitments, answers - /// presence probes with a yes, suppresses the replication that would repair it, and - /// earns a penalty at the next commitment-bound audit. Those penalties are not - /// suspended. - fn mark_suspect(&self, address: &XorName) { - if self.suspect.write().insert(*address) { - self.note_health_changed(); - warn!( - "Chunk {} is on disk but could not be read; this node stops answering for \ - it until a read succeeds", - hex::encode(address) - ); - } - } - - /// What the store's health looked like at this moment. - /// - /// Compare a value taken before a long-running check with one taken after, or after - /// taking a lock: different means a chunk stopped being servable in between and any - /// conclusion drawn from that check is out of date. - #[must_use] - pub fn health_generation(&self) -> u64 { - self.health.load(std::sync::atomic::Ordering::Acquire) - } - - /// Record that a chunk stopped being servable. - fn note_health_changed(&self) { - self.health - .fetch_add(1, std::sync::atomic::Ordering::AcqRel); - } - - /// Stop answering for a chunk a read has proven wrong. - /// - /// Unlike a chunk that merely could not be read, a later read does not clear this. - /// The bytes are wrong, and reading them again says the same thing; only replacing - /// them or removing them settles it. Bumping health matters as much as the suppression: a chunk that - /// has become unservable since the last pre-retirement pass must invalidate that pass, - /// or a repair that fails leaves the node deleting the copy it would have repaired - /// from. - /// - /// For callers outside this module that have proven it themselves. - pub fn note_known_wrong(&self, address: &XorName) { - self.mark_known_wrong(address); - } - - /// Stop answering for a chunk a read has proven wrong. - fn mark_known_wrong(&self, address: &XorName) { - if self.known_wrong.write().insert(*address) { - self.note_health_changed(); - warn!( - "Chunk {} does not match its name; this node stops answering for it until \ - it is repaired or removed", - hex::encode(address) - ); - } - } - - /// A caller outside this module has proven the stored bytes are right. - pub fn note_bytes_proven_good(&self, address: &XorName) { - self.clear_known_wrong(address); - self.clear_suspect(address); - } - - /// Answer for a chunk again, after it has been replaced or removed. - fn clear_known_wrong(&self, address: &XorName) { - self.known_wrong.write().remove(address); - } - - /// Answer for a chunk again, after a read that worked. - fn clear_suspect(&self, address: &XorName) { - if !self.suspect.read().contains(address) { - return; - } - if self.suspect.write().remove(address) { - info!( - "Chunk {} could be read again; this node answers for it once more", - hex::encode(address) - ); - } - } - - /// Number of chunks currently stored. - /// - /// The physical count: every name in the index, including chunks the node has stopped - /// answering for because a read found them wrong or could not read them at all. It is - /// deliberately not the same number as `all_keys().len()`, which is what the node is - /// willing to claim and so leaves those out. - /// - /// Anything asking "how much is on this disk" wants this one, and that is what its - /// callers ask: the migration's progress, the storage stats, and the size an audit is - /// built for. Anything asking "what will this node answer for" wants `all_keys`. - /// Quietly filtering this one would move all three of those without saying so, which - /// is why the difference is written down here rather than removed. - /// - /// # Errors - /// - /// Never fails. The signature matches the LMDB store's. - pub fn current_chunks(&self) -> Result { - Ok(self.index.read().len() as u64) - } - - /// Operation statistics, with the live chunk count filled in. - #[must_use] - pub fn stats(&self) -> StorageStats { - let mut stats = self.stats.read().clone(); - stats.current_chunks = self.index.read().len() as u64; - stats - } - - /// The node root directory this store was configured with. - #[must_use] - pub fn root_dir(&self) -> &Path { - &self.config.root_dir - } - - /// The directory holding the shard tree. - #[must_use] - pub fn chunks_dir(&self) -> &Path { - &self.chunks_dir - } - - /// Reject work early when the disk cannot take another chunk at all. - /// - /// # Errors - /// - /// Returns [`Error::Storage`] when free space is below the configured reserve. - pub fn check_capacity(&self) -> Result<()> { - self.capacity.check(0) - } - - /// Three-way answer to "can this store take a write right now". - /// - /// Kept distinct from [`Self::check_capacity`] because a failed free-space query and a - /// genuinely full disk are not the same thing, and the replication verification cycle - /// depends on the difference: a full disk is a standing condition worth minutes of - /// backoff, while a `statvfs` that failed says nothing about available space and may - /// well succeed on the next pass. - #[must_use] - pub fn capacity_verdict(&self) -> crate::storage::CapacityVerdict { - match self.capacity.measure_available() { - Some(available) if available < self.capacity.reserve => { - crate::storage::CapacityVerdict::Full - } - Some(_) => crate::storage::CapacityVerdict::Writable, - None => crate::storage::CapacityVerdict::Unknown, - } - } - - /// Reject work early when the disk cannot take `bytes` more. - /// - /// # Errors - /// - /// Returns [`Error::Storage`] when the write would not fit above the reserve. - pub fn check_capacity_for(&self, bytes: u64) -> Result<()> { - self.capacity.check(bytes) - } - - /// Force the next capacity question to re-measure the filesystem. - /// - /// Called after the legacy environment is removed, because that is a step change in - /// free space that the short-lived cache would otherwise hide for a few seconds. - pub fn invalidate_capacity_cache(&self) { - self.capacity.invalidate(); - } - - /// Test-only handle to the put gate. - /// - /// Hold the write half to park the next write inside its blocking closure, for - /// example to prove that shutdown waits for a write whose awaiter was dropped. - #[cfg(any(test, feature = "test-utils"))] - #[must_use] - pub fn test_put_gate(&self) -> Arc> { - Arc::clone(&self.test_put_gate) - } - - /// Register a write of `address` and hand back the token that clears it. - /// - /// The token must be moved into the blocking closure that does the work, so the entry - /// is cleared by the thread that finishes rather than by a caller that may be gone. - fn begin_write(&self, address: &XorName) -> WriteInFlight { - *self.writing.lock().entry(*address).or_insert(0) += 1; - WriteInFlight { - writing: Arc::clone(&self.writing), - finished: Arc::clone(&self.write_finished), - address: *address, - } - } - - /// Wait until nothing is part-way through writing `address`. - /// - /// For callers that must be last: a delete whose key still has a write in flight - /// would be undone by that write landing afterwards. - pub async fn wait_for_write(&self, address: &XorName) { - loop { - // Registered before the check, so a clear between the two is not missed. - let waiting = self.write_finished.notified(); - if !self.writing.lock().contains_key(address) { - return; - } - waiting.await; - } - } - - /// How many blocking tasks this store currently has in flight. Tests only. - /// - /// Lets a test wait for work to have actually started rather than guessing at a - /// delay, which is the difference between a test that proves something and one that - /// passes because the machine was quick. - #[cfg(test)] - #[must_use] - pub(crate) fn tasks_in_flight(&self) -> usize { - self.blocking_tracker.len() - } - - /// Wait until every blocking task this store spawned has finished. - /// - /// Dropping the awaiting future does not cancel a `spawn_blocking` closure, so - /// shutdown has to wait for the closure itself. - pub async fn wait_idle(&self) { - self.blocking_tracker.close(); - self.blocking_tracker.wait().await; - self.blocking_tracker.reopen(); - } - - /// Absolute path of a chunk file. - fn chunk_path(&self, address: &XorName) -> PathBuf { - self.chunks_dir - .join(shard_name(address)) - .join(hex::encode(address)) - } - - /// A temp name unique to this store instance, and distinguishable from a chunk name. - /// - /// The nonce matters: two `FileStore`s on one root in one process share a PID, and a - /// recycled PID collides with an age-gated leftover. Either way `create_new` would - /// fail and surface as a spurious write error. - fn next_temp_name(&self) -> String { - let seq = self.temp_seq.fetch_add(1, Ordering::Relaxed); - format!( - "{TEMP_PREFIX}{}.{:08x}.{seq}", - std::process::id(), - self.nonce - ) - } - - /// Read a chunk file, dropping the index entry if the file has vanished. - async fn read_file(&self, address: &XorName) -> Result>> { - let path = self.chunk_path(address); - let read = self - .blocking_tracker - .spawn_blocking(move || -> Result>> { - match open_regular(&path) { - Ok(Some(f)) => read_bounded(f, &path).map(Some), - Ok(None) => Ok(None), - Err(e) => Err(e), - } - }) - .await - .map_err(|e| Error::Storage(format!("Chunk store read task failed: {e}")))?; - - // Every read decides the question, not only the ones that were checking. A read - // that failed means this chunk cannot be served, whoever asked; a read that - // worked means it can be, whoever asked. Doing this anywhere else leaves a key - // stuck unadvertised after the fault has cleared, or advertised after it has not. - let read = match read { - Ok(read) => { - self.clear_suspect(address); - read - } - Err(e) => { - self.mark_suspect(address); - return Err(e); - } - }; - - if read.is_none() && self.forget_if_absent(address).await { - // The file went away underneath us. Stop advertising the key so the close - // group notices the shortfall and replication puts it back. - warn!( - "Chunk {} is indexed but its file is missing; dropped from the index so \ - replication can repair it", - hex::encode(address) - ); - } - Ok(read) - } - - /// Drop an index entry whose file is genuinely gone. - /// - /// Re-checks under the address's write lane, so a chunk republished between the - /// failing read and this call keeps its entry. - async fn forget_if_absent(&self, address: &XorName) -> bool { - // Not suspect any more: it is not unreadable, it is not there. - self.clear_suspect(address); - let path = self.chunk_path(address); - let lanes = Arc::clone(&self.write_lanes); - let index = Arc::clone(&self.index); - let lane = shard_index(address); - let key = *address; - // The bump happens inside the closure, with the mutation it describes. The - // closure runs to completion on its own thread whether or not anyone is still - // awaiting it, so bumping after the await is skipped entirely when a shutdown - // drops the caller — and the index change it was meant to announce still lands. - // A cached pre-retirement proof would then stay valid over a store that had - // quietly lost a chunk. - let health = Arc::clone(&self.health); - self.blocking_tracker - .spawn_blocking(move || { - let _lane = lanes.get(lane).map(parking_lot::Mutex::lock); - if path.exists() { - return false; - } - let forgotten = index.write().remove(&key); - if forgotten { - health.fetch_add(1, std::sync::atomic::Ordering::AcqRel); - } - forgotten - }) - .await - .unwrap_or(false) - } - - /// Remove a chunk whose bytes do not match its name, and stop advertising it. - /// - /// Re-reads and re-verifies under the address's write lane first. A read that failed - /// verification is rare enough that paying for one extra read is worth never - /// discarding a chunk that a concurrent write had already repaired. - async fn quarantine_corrupt(&self, address: &XorName) { - let path = self.chunk_path(address); - let lanes = Arc::clone(&self.write_lanes); - let index = Arc::clone(&self.index); - let lane = shard_index(address); - let key = *address; - // For the reason given on `forget_if_absent`: this closure outlives its awaiter, - // and the change it makes has to be announced by the same thread that makes it. - let health = Arc::clone(&self.health); - // And the store-lock lease, for the reason `put`, `repair`, `delete` and the - // startup scan carry it: this closure outlives its awaiter, so without it a - // cancelled verification whose caller dropped the last `FileStore` would unlink - // inside a directory a second process had already been handed. - let lease = Arc::clone(&self.lock); - let outcome = - self.blocking_tracker - .spawn_blocking(move || -> std::io::Result { - let _lease = lease; - let _lane = lanes.get(lane).map(parking_lot::Mutex::lock); - // Nothing is thrown away without proof. A re-read that fails says the - // question could not be answered this time, not that the bytes are wrong, - // and a repair may have published a good copy since the read that brought - // us here. Treating either as corruption deletes a chunk this node has. - let buf = match open_regular(&path) { - Ok(Some(f)) => read_bounded(f, &path) - .map_err(|e| std::io::Error::other(e.to_string()))?, - Ok(None) => { - index.write().remove(&key); - health.fetch_add(1, std::sync::atomic::Ordering::AcqRel); - return Ok(true); - } - Err(e) => return Err(std::io::Error::other(e.to_string())), - }; - if crate::client::compute_address(&buf) == key { - // Repaired between the failing read and now. Leave it alone. - return Ok(false); - } - std::fs::remove_file(&path)?; - // The same flush the ordinary delete does, for the same reason: an - // unlink that has not reached the directory can be undone by a power - // loss, and here the entry that comes back is one this node has proven - // wrong. The startup scan would re-index it by name, and the - // known-wrong mark that would otherwise hold it back lives only in - // memory and does not survive the restart, so the node would go back to - // claiming and committing to a chunk it already knows is bad. - if let Some(shard) = path.parent() { - fsync_dir_best_effort(shard); - } - index.write().remove(&key); - health.fetch_add(1, std::sync::atomic::Ordering::AcqRel); - Ok(true) - }) - .await; - match outcome { - Ok(Ok(true)) => { - self.clear_known_wrong(address); - self.clear_suspect(address); - warn!( - "Removed corrupt chunk file {}; replication will repair it", - hex::encode(address) - ); - } - Ok(Ok(false)) => { - // The re-read hashed and matched: a repair landed between the failing - // read and this one. - self.clear_known_wrong(address); - self.clear_suspect(address); - debug!( - "Chunk {} verified on re-read; leaving it in place", - hex::encode(address) - ); - } - // Still indexed, so it must not still be claimed: the read that brought us - // here proved the bytes wrong, and the node would otherwise go on committing - // to a chunk it knows it cannot serve. - Ok(Err(e)) => { - self.mark_suspect(address); - warn!( - "Corrupt chunk {} could not be removed: {e}. It stays on disk, and \ - this node stops answering for it.", - hex::encode(address) - ); - } - Err(e) => { - self.mark_suspect(address); - warn!("Corrupt-chunk removal task failed: {e}"); - } - } - } -} - -// ──────────────────────────────────────────────────────────────────────────── -// Free functions -// ──────────────────────────────────────────────────────────────────────────── - -/// Create the destination shard directory if this store has not seen it yet. -/// -/// A newly created directory entry is only durable once its parent is flushed; without -/// that a crash could take the directory and the chunk inside it together. -fn ensure_shard_dir( - chunks_dir: &Path, - dir: &Path, - shard: usize, - present: &parking_lot::Mutex<[bool; SHARD_COUNT]>, -) -> Result<()> { - if present.lock().get(shard).copied().unwrap_or(false) { - return Ok(()); - } - std::fs::create_dir_all(dir).map_err(|e| { - Error::Storage(format!( - "Failed to create shard directory {}: {e}", - dir.display() - )) - })?; - // Load-bearing, like the flush that publishes a chunk into this directory. Until the - // parent is flushed the shard's own entry can be lost, and losing it loses every chunk - // inside it. Reporting the shard present anyway would let the very first chunk written - // into it count as durably stored. - fsync_dir(chunks_dir).map_err(|e| { - Error::Storage(format!( - "Created shard directory {} but could not flush {}: {e}. Not marking the shard \ - usable, because a directory that is not durable cannot hold a chunk that is.", - dir.display(), - chunks_dir.display() - )) - })?; - if let Some(slot) = present.lock().get_mut(shard) { - *slot = true; - } - Ok(()) -} - -/// Shard directory index for an address: its last byte. -fn shard_index(address: &XorName) -> usize { - address.last().copied().unwrap_or(0) as usize -} - -/// Shard directory name for an address: the last two characters of its hex form. -fn shard_name(address: &XorName) -> String { - format!("{:02x}", shard_index(address)) -} - -/// True for a string of hex digits in either case. -fn is_hex_any_case(s: &str) -> bool { - !s.is_empty() && s.bytes().all(|b| b.is_ascii_hexdigit()) -} - -/// Move an entry aside under a name that can never be read as a chunk. -fn quarantine_entry(path: &Path) { - let aside = path.with_extension("not-a-chunk"); - match std::fs::rename(path, &aside) { - Ok(()) => warn!( - "Chunk store: moved {} aside to {}; a name that differs from a chunk name only \ - by case collides with it on Windows and macOS", - path.display(), - aside.display() - ), - Err(e) => warn!( - "Chunk store: {} collides with a chunk name by case folding and could not be \ - moved aside: {e}. Rename or delete it.", - path.display() - ), - } -} - -/// True for a string of lowercase hex digits only. -fn is_lower_hex(s: &str) -> bool { - !s.is_empty() && s.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')) -} - -/// Decode a filename back into the address it names, or `None` if it is not one. -/// -/// Rejects uppercase deliberately. On a case-folding filesystem (NTFS, default APFS) -/// accepting both cases would let one file answer to two index entries. -fn decode_chunk_name(name: &str) -> Option { - if name.len() != CHUNK_NAME_LEN || !is_lower_hex(name) { - return None; - } - let bytes = hex::decode(name).ok()?; - XorName::try_from(bytes.as_slice()).ok() -} - -/// Flush a directory and report whether it worked, for callers outside this module. -/// -/// For the one caller whose next step is destructive: retirement moves the legacy -/// environment aside and then deletes it under its new name, so if the rename has not -/// reached the disk when the delete lands, a power loss brings the environment back under -/// its old name with its contents gone. -/// -/// # Errors -/// -/// Returns the underlying I/O error. Off Unix there is no way to flush a directory through -/// the standard library, so this reports success without being able to promise anything. -pub fn fsync_path(path: &Path) -> std::io::Result<()> { - fsync_dir(path) -} - -/// Flush a directory so a rename or creation inside it survives power loss. -/// -/// Best effort by design. Linux and XFS require it, macOS accepts it with undocumented -/// effect, and Windows offers no way to do it at all through the standard library. The -/// content is content-addressed and re-replicable, so a lost directory entry costs a -/// refetch rather than data. Pretending otherwise in the code would be dishonest. -#[cfg(unix)] -fn fsync_dir_best_effort(path: &Path) { - if let Err(e) = fsync_dir(path) { - debug!("Directory flush of {} failed: {e}", path.display()); - } -} - -/// Flush a directory, reporting whether it worked. -/// -/// Used where the answer is load-bearing: a chunk copied out of the legacy store is only -/// durable once its directory entry is, and that copy is what permits the legacy store to -/// be deleted. -#[cfg(unix)] -fn fsync_dir(path: &Path) -> std::io::Result<()> { - File::open(path)?.sync_all() -} - -/// Off Unix there is no way to flush a directory through the standard library, so this -/// reports success without being able to promise anything. -/// -/// That is why the publish path off Unix does not use a rename at all: it creates the -/// chunk under its final name and flushes the file, which Microsoft documents as flushing -/// the creation metadata with it. Directory creation has no equivalent, so the guarantee -/// there rests on the pre-retirement pass, which re-reads every chunk before the legacy -/// store is deleted, and on the operator gate that keeps retirement off a platform until -/// forced power loss has been shown to hold old-or-new on it. -/// -/// Returns a `Result` so the callers that must handle a flush failure on Unix read the -/// same on every platform. -#[cfg(not(unix))] -#[allow(clippy::unnecessary_wraps)] -fn fsync_dir(_path: &Path) -> std::io::Result<()> { - Ok(()) -} - -/// No-op on platforms with no way to flush a directory handle. -#[cfg(not(unix))] -fn fsync_dir_best_effort(_path: &Path) {} - -/// Warn if the deepest chunk path this store can produce is close to `MAX_PATH`. -#[cfg(windows)] -fn check_path_budget(chunks_dir: &Path) { - // Measured absolute, because that is what the filesystem sees. A relative root is the - // case that still fails hard at MAX_PATH, since the standard library's long-path - // handling only applies to paths it resolves as absolute. - let absolute = if chunks_dir.is_absolute() { - chunks_dir.to_path_buf() - } else { - std::env::current_dir() - .map_or_else(|_| chunks_dir.to_path_buf(), |cwd| cwd.join(chunks_dir)) - }; - // `{chunks_dir}\{xy}\{64 hex}` — two separators, two shard characters, 64 name - // characters. - let deepest = absolute.as_os_str().len() + 1 + 2 + 1 + CHUNK_NAME_LEN; - if deepest > WINDOWS_PATH_WARN_LEN { - warn!( - "Chunk file paths will be {deepest} characters, close to the {} character \ - Windows limit. Move the node root closer to the drive letter if writes start \ - failing.", - WINDOWS_PATH_WARN_LEN - ); - } -} - -/// No-op where path length is not a practical constraint. -#[cfg(not(windows))] -fn check_path_budget(_chunks_dir: &Path) {} - -/// Write `bytes` to `path` durably, for small metadata files outside the shard tree. -/// -/// # Errors -/// -/// Returns [`Error::Storage`] if the file cannot be written or published. -pub fn write_file_durably(path: &Path, bytes: &[u8]) -> Result<()> { - write_file_atomic(path, bytes) -} - -/// Write `bytes` to `path` so a reader sees either the old content or the new. -/// Is this the exact name [`write_file_atomic`] gives its temporaries? -/// -/// `.tmp..<8 hex>.marker`, with both middle parts checked. Matching on the prefix and -/// suffix alone would also take `.tmp.operator-notes.marker`, and this runs over a -/// directory holding a node's data, so what it removes is not a place to be approximate. -fn is_marker_temp_name(name: &str) -> bool { - let Some(rest) = name.strip_prefix(TEMP_PREFIX) else { - return false; - }; - let Some(rest) = rest.strip_suffix(".marker") else { - return false; - }; - let mut parts = rest.split('.'); - let (Some(pid), Some(nonce), None) = (parts.next(), parts.next(), parts.next()) else { - return false; - }; - !pid.is_empty() - && pid.bytes().all(|b| b.is_ascii_digit()) - && nonce.len() == 8 - && nonce.bytes().all(|b| b.is_ascii_hexdigit()) -} - -/// Remove marker temporaries a previous run left beside `path`. -/// -/// [`write_file_atomic`] writes its temporary next to its target. For the layout marker -/// that is inside `chunks/`, which the startup scan sweeps; for the migration marker it is -/// the node root, which nothing sweeps, so a crash between the write and the rename leaves -/// one there for the life of the node. Each is a few hundred bytes, so this is inodes -/// rather than capacity, but nothing else was ever going to remove them. -/// -/// Only the exact shape this module writes, and only files: a name has to carry the temp -/// prefix and the marker suffix. Anything broader would be this function deciding what -/// else in a node's root directory is rubbish, which is not its business. -/// -/// Best effort throughout. Failing to tidy up is not a reason to refuse to start, and the -/// caller takes the store lock before this runs, so there is no other process whose live -/// temporary this could take. -pub(crate) fn sweep_marker_temps(dir: &Path) { - let Ok(entries) = std::fs::read_dir(dir) else { - return; - }; - for entry in entries.flatten() { - let name = entry.file_name(); - let Some(name) = name.to_str() else { - continue; - }; - if !is_marker_temp_name(name) { - continue; - } - if !entry.file_type().is_ok_and(|kind| kind.is_file()) { - continue; - } - match std::fs::remove_file(entry.path()) { - Ok(()) => debug!( - "Swept a leftover marker temporary {}", - entry.path().display() - ), - Err(e) => debug!("Could not sweep {}: {e}", entry.path().display()), - } - } -} - -fn write_file_atomic(path: &Path, bytes: &[u8]) -> Result<()> { - let Some(dir) = path.parent() else { - return Err(Error::Storage(format!( - "Refusing to write {} — it has no parent directory", - path.display() - ))); - }; - let temp = dir.join(format!( - "{TEMP_PREFIX}{}.{:08x}.marker", - std::process::id(), - rand::random::() - )); - write_temp(&temp, bytes)?; - // Through the retry, because these small files (the layout marker, the migration - // state) are rewritten while the node runs, and on Windows a scanner holding a handle - // for a few milliseconds turns an ordinary rewrite into a hard failure. - rename_with_retry(&temp, path).map_err(|e| { - let _ = std::fs::remove_file(&temp); - Error::Storage(format!("Failed to publish {}: {e}", path.display())) - })?; - fsync_dir_best_effort(dir); - Ok(()) -} - -/// Read the layout marker, writing the current one if the store is new. -fn read_or_write_layout(chunks_dir: &Path) -> Result { - let path = chunks_dir.join(LAYOUT_FILE_NAME); - match read_small_file(&path) { - Ok(bytes) => serde_json::from_slice(&bytes).map_err(|e| { - Error::Storage(format!( - "Chunk store layout marker {} is unreadable: {e}. Refusing to open rather \ - than guess the layout.", - path.display() - )) - }), - Err(e) if e.kind() == ErrorKind::NotFound => { - if store_has_entries(chunks_dir) { - warn!( - "Chunk store at {} has data but no layout marker. Adopting it under \ - the current scheme, which is the only one this build implements. If \ - it was written by a build with a different layout its chunks will \ - appear to be missing.", - chunks_dir.display() - ); - } - let layout = StoreLayout::default(); - let bytes = serde_json::to_vec_pretty(&layout) - .map_err(|e| Error::Storage(format!("Failed to encode chunk store layout: {e}")))?; - write_file_atomic(&path, &bytes)?; - debug!("Wrote chunk store layout marker to {}", path.display()); - Ok(layout) - } - Err(e) => Err(Error::Storage(format!( - "Failed to read chunk store layout marker {}: {e}", - path.display() - ))), - } -} - -/// Whether the store directory already holds at least one shard. -fn store_has_entries(chunks_dir: &Path) -> bool { - let Ok(entries) = std::fs::read_dir(chunks_dir) else { - return false; - }; - entries.filter_map(std::result::Result::ok).any(|e| { - e.file_name() - .to_str() - .is_some_and(|n| n.len() == 2 && is_lower_hex(n)) - }) -} - -/// Largest a metadata marker may be before it is treated as corrupt. -const MAX_MARKER_BYTES: u64 = 64 * 1024; - -/// Read a small metadata file, refusing an implausibly large one. -/// -/// The chunk path is bounded for exactly this reason; the markers live in the same data -/// directory and deserve the same ceiling. -/// -/// # Errors -/// -/// Returns an I/O error, including `NotFound`, so callers can distinguish "no marker yet". -pub fn read_small_file(path: &Path) -> std::io::Result> { - let file = File::open(path)?; - let mut bytes = Vec::new(); - let read = file.take(MAX_MARKER_BYTES + 1).read_to_end(&mut bytes)?; - if read as u64 > MAX_MARKER_BYTES { - return Err(std::io::Error::other(format!( - "{} is larger than the {MAX_MARKER_BYTES} byte limit for a marker file", - path.display() - ))); - } - Ok(bytes) -} - -/// Take the store lock, or refuse to open the store. -/// -/// Both failures are refusals, deliberately. Unlike LMDB, which was genuinely -/// multi-process safe, two of these stores on one directory keep independent in-memory -/// indices, independent views of what is in flight, and independent opinions about -/// whether the legacy environment may be deleted: both would report the same write as -/// new and each would keep serving keys the other had deleted. A node that cannot create -/// the lock file has no way to know it is alone, and this is the one migration where -/// being wrong about that destroys data. -/// -/// The lock is an [`Arc`] so the work that relies on it can hold a lease. The startup -/// scan sweeps interrupted writes on the strength of being alone in the directory, and it -/// runs on a thread that outlives the future that started it. -/// -/// # Errors -/// -/// Returns [`Error::Storage`] when another process owns the directory, or when the lock -/// file cannot be created. -fn acquire_store_lock(chunks_dir: &Path) -> Result> { - let path = chunks_dir.join(LOCK_FILE_NAME); - let file = match OpenOptions::new() - .write(true) - .create(true) - .truncate(false) - .open(&path) - { - Ok(f) => f, - // Not a warning and carry on. Without this lock two processes can open the same - // directory, each with its own index, its own view of what is in flight, and its - // own opinion about whether the legacy environment may be deleted. A node that - // cannot take it has no way to know it is alone, and this is the one migration - // where being wrong about that destroys data. - Err(e) => { - return Err(Error::Storage(format!( - "Could not create the chunk store lock {}: {e}. Refusing to start: \ - without it this node cannot tell whether another is using the same data \ - directory. Fix the permissions on that path, or remove a stale lock file \ - left by a different user.", - path.display() - ))) - } - }; - match file.try_lock_exclusive() { - Ok(()) => Ok(Arc::new(file)), - Err(e) => Err(Error::Storage(format!( - "Another process already has the chunk store at {} open ({e}). Two nodes \ - cannot share one data directory: each keeps its own index and they would \ - disagree about what is stored. Stop the other node first.", - chunks_dir.display() - ))), - } -} - -/// What a startup scan found. -struct ScanResult { - /// Every published address, ascending. - keys: Vec, - /// Which shard directories already exist. - shards_present: [bool; SHARD_COUNT], - /// Orphaned temp files removed. - swept_temps: usize, - /// Entries that were neither a chunk nor one of ours. - skipped: usize, -} - -/// Rebuild the key set from directory entries. -/// -/// Reads names only. A `stat` per entry costs about ten times the enumeration on Linux -/// and macOS and fifty to sixty times on Windows, and buys nothing: the filename is the -/// key, and the content is verified on read. -fn scan_store(chunks_dir: &Path) -> Result { - let mut result = ScanResult { - keys: Vec::new(), - shards_present: [false; SHARD_COUNT], - swept_temps: 0, - skipped: 0, - }; - - let top = std::fs::read_dir(chunks_dir).map_err(|e| { - Error::Storage(format!( - "Failed to enumerate chunk store {}: {e}", - chunks_dir.display() - )) - })?; - - for entry in top { - let entry = entry.map_err(|e| { - Error::Storage(format!( - "Failed to read an entry of {}: {e}", - chunks_dir.display() - )) - })?; - let name = entry.file_name(); - let Some(name) = name.to_str() else { - result.skipped = result.skipped.saturating_add(1); - continue; - }; - if name == LAYOUT_FILE_NAME || name == LOCK_FILE_NAME { - continue; - } - if name.starts_with(TEMP_PREFIX) { - if sweep_temp(&entry.path()) { - result.swept_temps = result.swept_temps.saturating_add(1); - } - continue; - } - if name.len() != 2 || !is_lower_hex(name) { - warn!( - "Chunk store: ignoring unexpected entry {name} in {}", - chunks_dir.display() - ); - result.skipped = result.skipped.saturating_add(1); - continue; - } - let Ok(shard) = u8::from_str_radix(name, 16) else { - result.skipped = result.skipped.saturating_add(1); - continue; - }; - // `shards_present` is set inside `scan_shard`, on success only. Setting it from - // the name alone would make a stray regular file called `ab` look like a shard - // that already exists, and every write to that shard would then fail with a - // misleading error until the node was restarted. - scan_shard(&entry.path(), shard, &mut result)?; - } - - result.keys.sort_unstable(); - result.keys.dedup(); - Ok(result) -} - -/// Scan one shard directory into `result`. -fn scan_shard(dir: &Path, shard: u8, result: &mut ScanResult) -> Result<()> { - let entries = match std::fs::read_dir(dir) { - Ok(e) => e, - // A stray file named like a shard, or a directory removed between the two reads. - // Neither is fatal, and neither marks the shard as present. - Err(e) if matches!(e.kind(), ErrorKind::NotFound | ErrorKind::NotADirectory) => { - warn!( - "Chunk store: {} is not a shard directory ({e}); ignoring it", - dir.display() - ); - result.skipped = result.skipped.saturating_add(1); - return Ok(()); - } - // Anything else is a real fault: a permission problem, exhausted descriptors, or - // failing hardware. Opening with a shard's worth of keys silently missing would - // make the node under-claim in its published commitment and stop serving chunks - // it still holds and is answerable for, so refuse to open at all. - Err(e) => { - return Err(Error::Storage(format!( - "Failed to enumerate shard {}: {e}. Refusing to open with an incomplete \ - key set.", - dir.display() - ))) - } - }; - if let Some(slot) = result.shards_present.get_mut(shard as usize) { - *slot = true; - } - - for entry in entries { - let entry = - entry.map_err(|e| Error::Storage(format!("Failed to read {}: {e}", dir.display())))?; - let name = entry.file_name(); - let Some(name) = name.to_str() else { - result.skipped = result.skipped.saturating_add(1); - continue; - }; - if name.starts_with(TEMP_PREFIX) { - if sweep_temp(&entry.path()) { - result.swept_temps = result.swept_temps.saturating_add(1); - } - continue; - } - let Some(key) = decode_chunk_name(name) else { - if name.len() == CHUNK_NAME_LEN && is_hex_any_case(name) { - // A case-folded twin of a real chunk name. On NTFS and default APFS the - // existence check in the write path folds onto it, so a paid write would - // be answered "already stored" and its bytes dropped. Move it aside. - quarantine_entry(&entry.path()); - } else { - warn!( - "Chunk store: ignoring non-chunk entry {name} in {}", - dir.display() - ); - } - result.skipped = result.skipped.saturating_add(1); - continue; - }; - // `file_type` comes from the directory entry itself on Linux and macOS and from - // the enumeration on Windows, so this is not the per-entry `stat` the scan - // deliberately avoids. A pipe, socket, device or directory wearing a chunk name - // must never enter the index: nothing downstream can read it, and it would sit in - // the published commitment forever. - match entry.file_type() { - Ok(kind) if kind.is_file() => {} - Ok(_) => { - warn!( - "Chunk store: {name} in {} is not a regular file; ignoring it", - dir.display() - ); - result.skipped = result.skipped.saturating_add(1); - continue; - } - // Not the same as knowing it is not a file. Treating an unanswered question - // as a no would drop a real chunk from the index and from the commitment - // while its bytes sit on disk, and the node would not serve it again until - // some later restart happened to succeed. Fail the scan instead: an index - // that is missing keys must never be published as this node's key set. - Err(e) => { - return Err(Error::Storage(format!( - "Could not tell what {name} in {} is: {e}. Refusing to publish an \ - index that may be missing chunks.", - dir.display() - ))); - } - } - // A file in the wrong shard is unreachable through `chunk_path`, so indexing it - // would make the index claim a key the read path cannot find. - if shard_index(&key) != shard as usize { - warn!( - "Chunk store: {name} is filed under shard {shard:02x} but belongs in {:02x}; \ - ignoring it. Move it or delete it.", - shard_index(&key) - ); - result.skipped = result.skipped.saturating_add(1); - continue; - } - result.keys.push(key); - } - Ok(()) -} - -/// Remove one orphaned temp file. Returns whether it went. -/// -/// Always removed. The scan that calls this runs only after the store lock has been taken, -/// so by then any temp file is an interrupted write of a previous run and there is no other -/// process that could be writing it. This used to describe a second, gentler mode for the -/// unlocked case; there was never any such branch and there is no caller that would need -/// one. -fn sweep_temp(path: &Path) -> bool { - match std::fs::remove_file(path) { - Ok(()) => { - debug!("Removed orphaned temporary file {}", path.display()); - true - } - Err(e) => { - debug!("Could not remove {}: {e}", path.display()); - false - } - } -} - -/// Open a chunk file, refusing anything that is not a regular file. -/// -/// `Ok(None)` means the file is not there. A named pipe wearing a valid chunk name would -/// otherwise block the opening thread forever: `open` on a FIFO with no writer does not -/// return, and enough of them would exhaust the blocking pool and stall every file and -/// database operation in the process. `O_NOFOLLOW` refuses a symlink for the same reason, -/// and both are checked on the handle rather than the path, so nothing can be swapped -/// underneath between the check and the open. -fn open_regular(path: &Path) -> Result> { - #[cfg(unix)] - let opened = { - use std::os::unix::fs::OpenOptionsExt; - OpenOptions::new() - .read(true) - .custom_flags(libc::O_NONBLOCK | libc::O_NOFOLLOW) - .open(path) - }; - #[cfg(not(unix))] - let opened = OpenOptions::new().read(true).open(path); - - let file = match opened { - Ok(f) => f, - Err(e) if e.kind() == ErrorKind::NotFound => return Ok(None), - Err(e) => { - return Err(Error::Storage(format!( - "Failed to open chunk file {}: {e}", - path.display() - ))) - } - }; - let is_regular = file.metadata().is_ok_and(|m| m.file_type().is_file()); - if !is_regular { - return Err(Error::Storage(format!( - "{} is not a regular file; refusing to read it as a chunk", - path.display() - ))); - } - Ok(Some(file)) -} - -/// Read a chunk file, refusing anything larger than a chunk can legitimately be. -/// -/// A corrupt, sparse, or locally planted file wearing a valid 64-hex name would -/// otherwise be read straight into memory, so a single bad entry could exhaust the node -/// during an ordinary GET or an audit response. -fn read_bounded(file: File, path: &Path) -> Result> { - let ceiling = MAX_CHUNK_SIZE as u64; - let mut buf = Vec::new(); - let read = file.take(ceiling + 1).read_to_end(&mut buf).map_err(|e| { - Error::Storage(format!("Failed to read chunk file {}: {e}", path.display())) - })?; - if read as u64 > ceiling { - return Err(Error::Storage(format!( - "Chunk file {} is larger than the {ceiling} byte maximum; refusing to read it", - path.display() - ))); - } - Ok(buf) -} - -/// Whether a Windows error is one a scanner or indexer holding a handle would produce. -/// -/// `ERROR_ACCESS_DENIED`, `ERROR_SHARING_VIOLATION`, `ERROR_LOCK_VIOLATION`. Every other -/// failure is deterministic and retrying it only burns a blocking thread. -fn is_windows_sharing_violation(e: &std::io::Error) -> bool { - matches!(e.raw_os_error(), Some(5 | 32 | 33)) -} - -/// Publish `temp_path` as `final_path`, retrying a transient sharing violation. -/// -/// On Windows an antivirus scanner or the search indexer can hold a handle to either -/// file for a few milliseconds after it is created, and `MoveFileEx` fails outright -/// rather than queueing. Retrying a bounded number of times turns that from a failed -/// write into a short pause. Every other error returns immediately. -fn rename_with_retry(temp_path: &Path, final_path: &Path) -> std::io::Result<()> { - let mut last = match std::fs::rename(temp_path, final_path) { - Ok(()) => return Ok(()), - Err(e) => e, - }; - if !cfg!(windows) || !is_windows_sharing_violation(&last) { - return Err(last); - } - for attempt in 1..=RENAME_RETRY_ATTEMPTS { - std::thread::sleep(RENAME_RETRY_BACKOFF * attempt); - match std::fs::rename(temp_path, final_path) { - Ok(()) => return Ok(()), - Err(e) => last = e, - } - } - Err(last) -} - -/// Write `payload` and publish it as `final_path`, replacing whatever is there. -/// -/// Success here means the bytes are durable, not merely written. The repair path this -/// serves runs during the pre-retirement pass, where a chunk that fails to match its -/// address is rewritten from the legacy store and the legacy store is then deleted. A -/// replacement that a power loss can undo would leave that chunk with the wrong bytes and -/// no other copy. -fn write_and_replace( - temp_path: &Path, - final_path: &Path, - payload: &[u8], - shard: &Path, -) -> Result<()> { - // Unix: an intra-directory rename is atomic, so a reader sees the old content or the - // new one and never an absence, and the directory flush is what makes it durable. - #[cfg(unix)] - { - write_temp(temp_path, payload)?; - if let Err(e) = rename_with_retry(temp_path, final_path) { - let _ = std::fs::remove_file(temp_path); - return Err(Error::Storage(format!( - "Failed to replace chunk {}: {e}", - final_path.display() - ))); - } - fsync_dir(shard).map_err(|e| { - Error::Storage(format!( - "Replaced {} but could not flush {}: {e}. Not reporting the repair as \ - done, because a rewrite that is not durable must not authorise deleting \ - the copy it was rewritten from.", - final_path.display(), - shard.display() - )) - })?; - Ok(()) - } - // Everywhere else, Windows included: there is no way to flush a directory through the - // standard library, so a rename cannot be shown to be durable at return. Overwriting - // the existing file changes no directory entry at all, and `sync_all` (FlushFileBuffers - // on Windows) is documented to flush the file's data, so a successful return is - // durable under a documented contract. - // - // The cost is that this is not atomic: a crash part-way leaves the file holding a mix - // of old and new bytes. That is safe here and only here, because the only caller that - // matters runs before the legacy store is deleted, and a crash means no report was - // produced and nothing was deleted. The next start re-reads the file, sees it does not - // match its address, and repairs it again from the store that is still there. - #[cfg(not(unix))] - { - let _ = temp_path; - let _ = shard; - let mut file = OpenOptions::new() - .write(true) - .truncate(true) - .open(final_path) - .map_err(|e| { - Error::Storage(format!( - "Failed to open {} for replacement: {e}", - final_path.display() - )) - })?; - file.write_all(payload).map_err(|e| { - Error::Storage(format!("Failed to rewrite {}: {e}", final_path.display())) - })?; - file.sync_all().map_err(|e| { - Error::Storage(format!( - "Rewrote {} but could not flush it: {e}. Not reporting the repair as \ - done, because a rewrite that is not durable must not authorise deleting \ - the copy it was rewritten from.", - final_path.display() - )) - })?; - Ok(()) - } -} - -/// Create `temp_path`, write `payload` into it, and flush it. -/// -/// Flushed before any rename. On ext4 `auto_da_alloc` only orders the data before the -/// rename's own commit; it does not make the data durable, and btrfs has been observed -/// reordering. A name must never become visible on bytes that are not on the platter. -fn write_temp(temp_path: &Path, payload: &[u8]) -> Result<()> { - let mut f = OpenOptions::new() - .write(true) - .create_new(true) - .open(temp_path) - .map_err(|e| { - Error::Storage(format!( - "Failed to create temporary file {}: {e}", - temp_path.display() - )) - })?; - if let Err(e) = f.write_all(payload) { - let _ = std::fs::remove_file(temp_path); - return Err(Error::Storage(format!( - "Failed to write {}: {e}", - temp_path.display() - ))); - } - if let Err(e) = f.sync_all() { - let _ = std::fs::remove_file(temp_path); - return Err(Error::Storage(format!( - "Failed to flush {}: {e}", - temp_path.display() - ))); - } - Ok(()) -} - -/// Write `payload` and publish it under `final_path`. -/// -/// The temp lives in the destination directory, so the publish is an intra-directory -/// rename: atomic on every filesystem we support, and needing only that one directory -/// Put `payload` on disk as `final_path`, durably. -/// -/// Returns [`PutOutcome::Duplicate`] when the name is already taken. The name is a hash -/// of the content, so that is not treated as proof the bytes are right: the caller -/// re-reads and verifies them. -#[cfg(unix)] -fn publish( - temp_path: &Path, - final_path: &Path, - payload: &[u8], - shard: &Path, -) -> std::result::Result { - // On Unix nothing is ever created under the final name by a failing path: the bytes go - // to a temporary and only a successful rename gives them the real name. So every - // failure here leaves the name as it found it. - publish_via_rename(temp_path, final_path, payload, shard) - .map_err(PublishFailed::nothing_written) -} - -/// Put `payload` on disk as `final_path`, durably. See [`publish_in_place`] for why this -/// takes a different route off Unix. -#[cfg(not(unix))] -fn publish( - temp_path: &Path, - final_path: &Path, - payload: &[u8], - shard: &Path, -) -> std::result::Result { - let _ = temp_path; - let _ = shard; - publish_in_place(final_path, payload) -} - -/// Create the chunk under its final name and flush it. Everywhere but Unix. -/// -/// There is no way to flush a directory through the standard library, and Microsoft does -/// not document `MoveFileEx` as durable at return unless it is called with -/// `MOVEFILE_WRITE_THROUGH`, which std does not use. So off Unix a rename cannot be -/// relied on to have reached the disk before the legacy store is deleted. -/// -/// Creating the file under its final name sidesteps the rename entirely. Microsoft -/// documents that creation metadata is cached and that `FlushFileBuffers`, which -/// `sync_all` calls on Windows, is the way to flush it. So a successful create, write and -/// flush is a durable publication under a documented contract, with no directory flush -/// and no rename involved. -/// -/// The cost is that a crash mid-write leaves a partial file wearing a real chunk name. -/// That is why a duplicate re-reads and verifies rather than trusting the name, and why -/// the pre-retirement pass re-hashes everything before anything is deleted. -#[cfg(not(unix))] -fn publish_in_place( - final_path: &Path, - payload: &[u8], -) -> std::result::Result { - // Test-only, and here rather than after the write so that it means the same thing on - // both platforms: the file half of a dual write has not happened yet. On Unix the - // equivalent point is the temporary file written and the rename not yet made, which is - // also before the chunk's name exists on disk. Stopping after the write instead would - // put the file under its real name already, so a crash there is not between the two - // halves at all, and it could not demonstrate anything about the missing flush either: - // killing a process does not empty the page cache, so the bytes are still there to be - // read. Only losing power loses them, which no test that kills a process can stage. - #[cfg(any(test, feature = "test-utils"))] - halt_here_if_asked(HALT_BEFORE_PUBLISH, final_path); - let mut file = match OpenOptions::new() - .write(true) - .create_new(true) - .open(final_path) - { - Ok(f) => f, - // Someone got there first. Immutable content under a content-addressed name, so - // the caller verifies what is already there rather than assuming it is right. - Err(e) if e.kind() == ErrorKind::AlreadyExists => return Ok(PutOutcome::Duplicate), - Err(e) => { - // Nothing was created, so nothing was spent. - return Err(PublishFailed::nothing_written(Error::Storage(format!( - "Failed to create chunk {}: {e}", - final_path.display() - )))); - } - }; - if let Err(e) = file.write_all(payload) { - drop(file); - // Taken back if it can be. Whether it could is what the caller needs: the file - // was created by this call, so if it is still there the space is spent. - let left_behind = std::fs::remove_file(final_path).is_err(); - return Err(PublishFailed { - error: Error::Storage(format!("Failed to write {}: {e}", final_path.display())), - left_behind, - }); - } - if let Err(e) = file.sync_all() { - drop(file); - // Taken back if it can be. Whether it could is what the caller needs: the file - // was created by this call, so if it is still there the space is spent. - let left_behind = std::fs::remove_file(final_path).is_err(); - return Err(PublishFailed { - error: Error::Storage(format!("Failed to flush {}: {e}", final_path.display())), - left_behind, - }); - } - Ok(PutOutcome::New) -} - -/// Write a temp beside the target and rename it into place. Unix only. -/// -/// Places the bytes and nothing more. Making the name durable is -/// [`flush_publication`]'s job, kept separate so a caller can tell a publish that spent no -/// space from one that spent it and could not be reported. -#[cfg(unix)] -fn publish_via_rename( - temp_path: &Path, - final_path: &Path, - payload: &[u8], - _shard: &Path, -) -> Result { - // Content is immutable and the name is its hash, so an existing file already holds - // exactly these bytes. Skipping the write is both cheaper and safer than replacing - // it: on Windows a rename over a file another thread has open fails outright. - // - // The caller flushes either way. A name that is already there is not proof it is - // durable: - // the write that put it there may have been this store's own previous attempt, whose - // rename landed and whose directory flush then failed. That attempt returned an - // error, so nothing was retired on the strength of it, but if this call reported a - // durable duplicate without flushing, the retry would silently launder an unflushed - // rename into a copy that authorises deleting the last other one. - let outcome = if final_path.exists() { - PutOutcome::Duplicate - } else { - write_temp(temp_path, payload)?; - // Test-only: the one moment a complete chunk exists on disk under a name nothing - // looks for. A crash test needs to die at a named point rather than wherever a - // sleep in another process happened to land. - #[cfg(any(test, feature = "test-utils"))] - halt_here_if_asked(HALT_BEFORE_PUBLISH, temp_path); - match rename_with_retry(temp_path, final_path) { - Ok(()) => PutOutcome::New, - Err(e) => { - let _ = std::fs::remove_file(temp_path); - // Another writer of the same address won the race, or the destination was - // open. Either way the bytes are already published. - if !final_path.exists() { - return Err(Error::Storage(format!( - "Failed to publish chunk {}: {e}", - final_path.display() - ))); - } - PutOutcome::Duplicate - } - } - }; - - Ok(outcome) -} - -/// A publish that failed, and whether it left its bytes on the disk. -/// -/// The second half is the point. A failure before anything was created has spent nothing; -/// one that created the file and then could not remove it again has spent the space, and -/// whoever is accounting for free space has to know which happened. Only the code that did -/// the creating can say. -struct PublishFailed { - error: Error, - left_behind: bool, -} - -impl PublishFailed { - /// A failure that created nothing. - fn nothing_written(error: Error) -> Self { - Self { - error, - left_behind: false, - } - } -} - -/// Make a publication durable by flushing the directory its name lives in. -/// -/// Separate from placing the bytes, because the caller has to tell the two failures apart. -/// A publish that fails before the bytes land has spent nothing; one that fails here has -/// spent the space and must not be reported as stored, so whoever is accounting for free -/// space has to charge it while whoever is accounting for chunks must not count it. -/// -/// NOT best effort. The directory flush is what makes the rename durable, and a copy -/// reported successful is what authorises deleting the only other copy. Swallowing the -/// failure would let a power loss discard the directory entry after the legacy store had -/// already been removed. -/// -/// # Errors -/// -/// Returns [`Error::Storage`] if the directory cannot be flushed. -#[cfg(unix)] -fn flush_publication(final_path: &Path, shard: &Path) -> Result<()> { - fsync_dir(shard).map_err(|e| { - Error::Storage(format!( - "Published {} but could not flush {}: {e}. Not reporting this chunk as stored, \ - because a copy that is not durable must not authorise deleting another.", - final_path.display(), - shard.display() - )) - }) -} - -/// Nothing to do off Unix, where the chunk is created under its final name and flushed -/// with `sync_all`, which is documented to carry its creation metadata with it, and where -/// there is no way to flush a directory at all. -#[cfg(not(unix))] -fn flush_publication(_final_path: &Path, _shard: &Path) -> Result<()> { - Ok(()) -} - -#[cfg(test)] -#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] -mod tests { - use super::*; - use std::collections::HashSet; - - /// A directory flush that fails must say so. - /// - /// The quiet version of this function is only used where the answer does not change - /// what happens next. On the publish path it does. - #[cfg(unix)] - #[test] - fn flushing_a_directory_that_is_not_there_reports_the_failure() { - let dir = TempDir::new().expect("temp dir"); - assert!(fsync_dir(dir.path()).is_ok()); - assert!(fsync_dir(&dir.path().join("no-such-shard")).is_err()); - } - - /// A chunk whose directory entry was never flushed is not reported as stored. - /// - /// This is the whole safety argument for retirement: the legacy store is deleted - /// because every chunk was copied durably. A published file whose directory flush - /// failed can vanish on power loss, so counting it as copied would lose data. The - /// file staying on disk afterwards is fine, the next pass republishes it. - #[cfg(unix)] - #[test] - fn a_publish_whose_directory_flush_fails_is_not_reported_as_stored() { - let dir = TempDir::new().expect("temp dir"); - let temp_path = dir.path().join("chunk.tmp"); - let final_path = dir.path().join("chunk"); - let unflushable = dir.path().join("shard-that-does-not-exist"); - - // Asserted in two steps, not chained. Chaining them means a regression in placing - // the bytes also produces an error, and the test passes without the flush ever - // being reached: it would be checking that something went wrong rather than that - // this went wrong. - let placed = publish_via_rename(&temp_path, &final_path, b"payload", &unflushable); - assert!( - placed.is_ok(), - "the bytes must be placed before this can be about the flush: {:?}", - placed.err() - ); - let outcome = flush_publication(&final_path, &unflushable); - - assert!( - outcome.is_err(), - "an unflushed publication must not be reported as stored" - ); - assert!( - !temp_path.exists(), - "the temp file must not be left behind either way" - ); - } - - use tempfile::TempDir; - - /// Open a store on a fresh temp directory with the disk reserve disabled. - async fn test_store() -> (FileStore, TempDir) { - let dir = TempDir::new().expect("temp dir"); - let store = FileStore::new(FileStoreConfig { - root_dir: dir.path().to_path_buf(), - verify_on_read: true, - disk_reserve: 0, - }) - .await - .expect("open store"); - (store, dir) - } - - /// Open a store on an existing directory, as a restart would. - async fn reopen(dir: &TempDir) -> FileStore { - FileStore::new(FileStoreConfig { - root_dir: dir.path().to_path_buf(), - verify_on_read: true, - disk_reserve: 0, - }) - .await - .expect("reopen store") - } - - /// An ordinary read settles whether the node answers for a chunk. - /// - /// Not only the reads that were checking something. A read that failed means the - /// chunk cannot be served, whoever asked; a read that worked means it can be. Deciding - /// this anywhere else leaves a key stuck unadvertised after the fault has cleared, or - /// advertised after it has not. - #[cfg(unix)] - #[tokio::test] - async fn an_ordinary_read_decides_whether_the_node_answers_for_a_chunk() { - use std::os::unix::fs::PermissionsExt; - - let (store, dir) = test_store().await; - let (addr, content) = addressed("read-decides"); - store.put(&addr, &content).await.expect("put"); - let path = store.chunk_path(&addr); - - let mut perms = std::fs::metadata(&path).expect("meta").permissions(); - perms.set_mode(0o000); - std::fs::set_permissions(&path, perms).expect("chmod"); - - assert!(store.get(&addr).await.is_err(), "the read must fail"); - assert!( - !store.exists(&addr).expect("exists"), - "and a plain read that failed must stop the node answering for it" - ); - - let mut perms = std::fs::metadata(&path).expect("meta").permissions(); - perms.set_mode(0o600); - std::fs::set_permissions(&path, perms).expect("chmod back"); - - assert_eq!( - store.get(&addr).await.expect("get").expect("present"), - content - ); - assert!( - store.exists(&addr).expect("exists"), - "and a plain read that worked must start it answering again" - ); - drop(dir); - } - - /// Two writes for one key: waiting means waiting for both. - /// - /// Cancellation releases the caller's lane while the blocking half survives, so a - /// second write for the same key can start behind the first. If the registry only - /// recorded that *something* was writing, whichever finished first would clear it and - /// a delete would be told the key was free while the other was still queued, then be - /// undone by it. - #[tokio::test] - async fn waiting_for_a_key_waits_for_every_write_of_it() { - let (store, dir) = test_store().await; - let store = Arc::new(store); - let (addr, content) = addressed("two-writers"); - - // Two registrations, as two overlapping writes would make. - let first = store.begin_write(&addr); - let second = store.begin_write(&addr); - - let waiting = { - let store = Arc::clone(&store); - tokio::spawn(async move { store.wait_for_write(&addr).await }) - }; - tokio::time::sleep(Duration::from_millis(50)).await; - assert!(!waiting.is_finished()); - - // One finishes. The other has not, so the wait must continue. - drop(first); - tokio::time::sleep(Duration::from_millis(50)).await; - assert!( - !waiting.is_finished(), - "one write finishing does not mean the key is free" - ); - - drop(second); - waiting - .await - .expect("the wait ends once both have finished"); - - // And the store is still usable afterwards. - store.put(&addr, &content).await.expect("put"); - assert!(store.exists(&addr).expect("exists")); - drop(dir); - } - - /// A chunk this store cannot read is kept but not claimed. - /// - /// Both halves matter. Deleting it, or dropping it from the index, is how a chunk ends - /// up in neither this store's view nor the legacy one, which is what retirement - /// destroys. Claiming it anyway puts the key in signed commitments and answers - /// presence probes with a yes for a chunk the node cannot serve, and the audit that - /// catches that still penalises. - #[cfg(unix)] - #[tokio::test] - async fn a_chunk_that_cannot_be_read_is_kept_but_not_claimed() { - use std::os::unix::fs::PermissionsExt; - - let (store, dir) = test_store().await; - let (addr, content) = addressed("unreadable-for-now"); - store.put(&addr, &content).await.expect("put"); - assert!(store.exists(&addr).expect("exists")); - - let path = store.chunk_path(&addr); - let mut perms = std::fs::metadata(&path).expect("meta").permissions(); - perms.set_mode(0o000); - std::fs::set_permissions(&path, perms).expect("chmod"); - - // Offering the same bytes again must not be acknowledged, and must not replace - // what is there on the strength of a read that did not happen. - assert!( - store.put(&addr, &content).await.is_err(), - "an unreadable chunk must not be reported as stored" - ); - assert!(path.exists(), "and the file must be left alone"); - assert!( - !store.exists(&addr).expect("exists"), - "but the node must stop claiming it" - ); - assert!(!store.all_keys().await.expect("keys").contains(&addr)); - - // Readable again: the node answers for it once more. - let mut perms = std::fs::metadata(&path).expect("meta").permissions(); - perms.set_mode(0o600); - std::fs::set_permissions(&path, perms).expect("chmod back"); - assert!(!store.put(&addr, &content).await.expect("put again")); - assert!(store.exists(&addr).expect("exists")); - assert!(store.all_keys().await.expect("keys").contains(&addr)); - drop(dir); - } - - /// Content plus the address it hashes to. - fn addressed(seed: &str) -> (XorName, Vec) { - let content = format!("chunk-content-{seed}").into_bytes(); - (crate::client::compute_address(&content), content) - } - - #[tokio::test] - async fn put_then_get_returns_the_same_bytes() { - let (store, _dir) = test_store().await; - let (addr, content) = addressed("a"); - - assert!(store.put(&addr, &content).await.expect("put")); - let got = store.get(&addr).await.expect("get").expect("present"); - assert_eq!(got, content); - } - - #[tokio::test] - async fn a_second_put_of_the_same_chunk_reports_not_new() { - let (store, _dir) = test_store().await; - let (addr, content) = addressed("b"); - - assert!(store.put(&addr, &content).await.expect("first put")); - assert!(!store.put(&addr, &content).await.expect("second put")); - assert_eq!(store.current_chunks().expect("count"), 1); - assert_eq!(store.stats().duplicates, 1); - } - - #[tokio::test] - async fn get_of_an_unknown_address_is_none() { - let (store, _dir) = test_store().await; - let (addr, _) = addressed("missing"); - assert!(store.get(&addr).await.expect("get").is_none()); - } - - #[tokio::test] - async fn exists_tracks_the_store() { - let (store, _dir) = test_store().await; - let (addr, content) = addressed("c"); - - assert!(!store.exists(&addr).expect("exists")); - store.put(&addr, &content).await.expect("put"); - assert!(store.exists(&addr).expect("exists")); - store.delete(&addr).await.expect("delete"); - assert!(!store.exists(&addr).expect("exists")); - } - - #[tokio::test] - async fn delete_unlinks_the_file_and_returns_the_space() { - let (store, _dir) = test_store().await; - let (addr, content) = addressed("d"); - store.put(&addr, &content).await.expect("put"); - - let path = store.chunk_path(&addr); - assert!(path.exists(), "the chunk file should be on disk"); - - assert!(store.delete(&addr).await.expect("delete")); - assert!(!path.exists(), "delete must actually unlink the file"); - assert_eq!(store.current_chunks().expect("count"), 0); - - // Deleting again is a no-op that reports nothing was there. - assert!(!store.delete(&addr).await.expect("second delete")); - } - - #[tokio::test] - async fn content_that_does_not_hash_to_its_address_is_rejected() { - let (store, _dir) = test_store().await; - let (addr, _) = addressed("e"); - let err = store - .put(&addr, b"different content") - .await - .expect_err("must reject"); - assert!( - format!("{err}").contains("Content address mismatch"), - "unexpected error: {err}" - ); - assert_eq!(store.current_chunks().expect("count"), 0); - } - - #[tokio::test] - async fn a_chunk_is_filed_under_the_last_two_hex_characters_of_its_address() { - let (store, dir) = test_store().await; - let (addr, content) = addressed("f"); - store.put(&addr, &content).await.expect("put"); - - let name = hex::encode(addr); - let expected_shard = name - .get(name.len() - 2..) - .expect("64-character name") - .to_string(); - let path = dir - .path() - .join(CHUNKS_DIR_NAME) - .join(&expected_shard) - .join(&name); - assert!(path.exists(), "expected the chunk at {}", path.display()); - } - - #[tokio::test] - async fn the_index_is_rebuilt_from_the_filesystem_on_restart() { - let (store, dir) = test_store().await; - let mut written = Vec::new(); - for i in 0..64 { - let (addr, content) = addressed(&format!("restart-{i}")); - store.put(&addr, &content).await.expect("put"); - written.push(addr); - } - drop(store); - - let reopened = reopen(&dir).await; - assert_eq!(reopened.current_chunks().expect("count"), 64); - for addr in &written { - assert!(reopened.exists(addr).expect("exists"), "lost a key"); - } - } - - #[tokio::test] - async fn all_keys_is_sorted_ascending() { - let (store, dir) = test_store().await; - for i in 0..128 { - let (addr, content) = addressed(&format!("sorted-{i}")); - store.put(&addr, &content).await.expect("put"); - } - - let keys = store.all_keys().await.expect("all_keys"); - let mut sorted = keys.clone(); - sorted.sort_unstable(); - assert_eq!(keys, sorted, "all_keys() must be ordered"); - - // And the order has to survive a restart, because the commitment builder - // truncates the responsible subset before the Merkle tree sorts it. - drop(store); - let reopened = reopen(&dir).await; - assert_eq!(reopened.all_keys().await.expect("all_keys"), keys); - } - - #[tokio::test] - async fn get_raw_skips_verification() { - let (store, _dir) = test_store().await; - let (addr, content) = addressed("raw"); - store.put(&addr, &content).await.expect("put"); - - // Corrupt the file behind the store's back. - std::fs::write(store.chunk_path(&addr), b"tampered").expect("tamper"); - - let raw = store.get_raw(&addr).await.expect("get_raw").expect("bytes"); - assert_eq!(raw, b"tampered"); - } - - /// A put whose caller goes away does not admit a key on bytes nothing has read. - /// - /// The blocking half of a put outlives the future that started it, deliberately, so - /// the work is never left half done. That makes anything it writes to memory a claim - /// the node keeps whether or not the caller is still there to finish checking it. - /// - /// For a chunk this call published the claim is earned: the bytes were hashed against - /// their own name on the way in. For a name that was already taken it is not. The - /// check that decides whether those bytes are good runs after the await, and a dropped - /// future skips it, so admitting the key in the closure claims a chunk nobody read. - /// - /// Staged with a fifo, which is the sharpest case and a real one: the startup scan - /// refuses non-regular entries by design, so this is a key the store has already - /// decided it must not claim, walked in through the back door. - #[cfg(unix)] - #[tokio::test] - // The gate is held across an await deliberately: holding it is what parks the put - // inside its closure, which is the state under test. Dropping it before awaiting would - // let the put finish and there would be nothing to cancel. - #[allow(clippy::await_holding_lock)] - async fn a_cancelled_put_does_not_admit_a_key_whose_bytes_were_never_read() { - let dir = TempDir::new().expect("temp dir"); - let store = Arc::new(reopen(&dir).await); - - // A name a real chunk would use, wearing something that is not a chunk. - let content = b"the bytes that belong under this name".to_vec(); - let addr = crate::client::compute_address(&content); - let shard = dir - .path() - .join(CHUNKS_DIR_NAME) - .join(format!("{:02x}", addr[31])); - std::fs::create_dir_all(&shard).expect("mkdir"); - let path = shard.join(hex::encode(addr)); - let name = std::ffi::CString::new(path.as_os_str().as_encoded_bytes()) - .expect("a path with no interior nul"); - // SAFETY: `name` is a valid NUL-terminated C string that outlives the call, and the - // mode is a constant. `mkfifo` reads the pointer and returns; nothing is retained. - #[allow(clippy::undocumented_unsafe_blocks, unsafe_code)] - let made = unsafe { libc::mkfifo(name.as_ptr(), 0o644) }; - assert_eq!(made, 0, "could not make the fifo this test needs"); - - // Hold the gate so the put parks inside the closure, then drop the future while it - // is parked. That is a caller going away mid-put, which is what a cancelled - // request, a client disconnect or a shutdown all look like from in here. - let gate = store.test_put_gate(); - let held = gate.write(); - let put = { - let store = Arc::clone(&store); - let content = content.clone(); - tokio::spawn(async move { store.put(&addr, &content).await }) - }; - // Waited for rather than slept at. A sleep proves nothing: if the put had not - // reached the gated closure yet, aborting would cancel it before it ever got - // there and the test would pass having staged nothing. - let deadline = std::time::Instant::now() + Duration::from_secs(30); - while store.tasks_in_flight() == 0 { - assert!( - std::time::Instant::now() < deadline, - "the put never reached the closure, so there was nothing to cancel" - ); - tokio::time::sleep(Duration::from_millis(5)).await; - } - put.abort(); - let _ = put.await; - drop(held); - store.wait_idle().await; - - assert!( - !store.is_indexed(&addr), - "a cancelled put admitted {} on bytes nothing read; the fifo under that name \ - would then be advertised, committed to, and audited against", - hex::encode(addr) - ); - assert!( - !store.exists(&addr).unwrap_or(true), - "and the node must not claim it either" - ); - } - - /// A marker temporary left in the node root is swept, and nothing else is. - /// - /// The migration marker is written next to itself in the root, which no sweep looked - /// at, so a crash between its write and its rename left one there for the life of the - /// node. Small, but nothing was ever going to remove it. - /// - /// The second half is the point: this runs over a directory holding a node's data, so - /// it has to take only the exact shape this module writes and leave everything else - /// where it is. - #[tokio::test] - async fn a_leftover_marker_temporary_is_swept_and_its_neighbours_are_not() { - let dir = TempDir::new().expect("temp dir"); - let root = dir.path(); - let leftover = root.join(format!("{TEMP_PREFIX}1234.abcdef01.marker")); - std::fs::write(&leftover, b"an interrupted marker write").expect("plant"); - - // Things that must survive: the marker itself, a chunk-shaped temp that belongs to - // the chunk tree's own sweep, and anything an operator put there. - let keep = [ - root.join("migration-state.json"), - root.join(format!("{TEMP_PREFIX}1234.abcdef01.chunk")), - root.join("notes.txt"), - // Prefix and suffix alone would take these. The pid and the nonce are checked - // because this runs over a directory holding a node's data. - root.join(format!("{TEMP_PREFIX}operator-notes.marker")), - root.join(format!("{TEMP_PREFIX}1234.nothex01.marker")), - root.join(format!("{TEMP_PREFIX}1234.abcdef0.marker")), - root.join(format!("{TEMP_PREFIX}1234.abcdef01.extra.marker")), - ]; - for path in &keep { - std::fs::write(path, b"keep me").expect("plant"); - } - - let store = reopen(&dir).await; - drop(store); - - assert!( - !leftover.exists(), - "the leftover marker temporary is still in the node root" - ); - for path in &keep { - assert!( - path.exists(), - "{} was swept and should not have been", - path.display() - ); - } - } - - #[tokio::test] - async fn a_corrupt_chunk_is_removed_so_replication_can_repair_it() { - let (store, _dir) = test_store().await; - let (addr, content) = addressed("corrupt"); - store.put(&addr, &content).await.expect("put"); - std::fs::write(store.chunk_path(&addr), b"tampered").expect("tamper"); - - let err = store.get(&addr).await.expect_err("verification must fail"); - assert!(format!("{err}").contains("verification failed"), "{err}"); - - assert!(!store.chunk_path(&addr).exists(), "corrupt file must go"); - assert!(!store.exists(&addr).expect("exists")); - assert!( - !store.all_keys().await.expect("all_keys").contains(&addr), - "a corrupt chunk must stop being advertised" - ); - assert_eq!(store.stats().verification_failures, 1); - } - - #[tokio::test] - async fn a_file_removed_underneath_the_store_drops_out_of_the_index() { - let (store, _dir) = test_store().await; - let (addr, content) = addressed("vanished"); - store.put(&addr, &content).await.expect("put"); - - std::fs::remove_file(store.chunk_path(&addr)).expect("remove behind our back"); - - assert!(store.get(&addr).await.expect("get").is_none()); - assert!(!store.exists(&addr).expect("exists")); - assert_eq!(store.current_chunks().expect("count"), 0); - } - - #[tokio::test] - async fn interrupted_writes_are_swept_at_startup() { - let (store, dir) = test_store().await; - let (addr, content) = addressed("sweep"); - store.put(&addr, &content).await.expect("put"); - let shard = store - .chunk_path(&addr) - .parent() - .expect("shard") - .to_path_buf(); - drop(store); - - let orphan = shard.join(format!("{TEMP_PREFIX}999.7")); - std::fs::write(&orphan, b"half a chunk").expect("write orphan"); - let stray_root = dir - .path() - .join(CHUNKS_DIR_NAME) - .join(format!("{TEMP_PREFIX}999.8")); - std::fs::write(&stray_root, b"half a marker").expect("write stray"); - - let reopened = reopen(&dir).await; - assert!(!orphan.exists(), "an interrupted write must not survive"); - assert!(!stray_root.exists(), "nor one at the store root"); - assert_eq!(reopened.current_chunks().expect("count"), 1); - } - - #[tokio::test] - async fn concurrent_writers_of_one_address_store_it_exactly_once() { - let (store, _dir) = test_store().await; - let store = Arc::new(store); - let (addr, content) = addressed("racing"); - - let mut tasks = Vec::new(); - for _ in 0..16 { - let store = Arc::clone(&store); - let content = content.clone(); - tasks.push(tokio::spawn( - async move { store.put(&addr, &content).await }, - )); - } - - let mut new_count = 0; - for task in tasks { - if task.await.expect("join").expect("put") { - new_count += 1; - } - } - assert_eq!(new_count, 1, "exactly one writer may report a new chunk"); - assert_eq!(store.current_chunks().expect("count"), 1); - assert_eq!( - store.get(&addr).await.expect("get").expect("present"), - content - ); - } - - #[tokio::test] - async fn names_that_are_not_lowercase_hex_are_ignored_by_the_scan() { - let (store, dir) = test_store().await; - let (addr, content) = addressed("scan"); - store.put(&addr, &content).await.expect("put"); - let shard = store - .chunk_path(&addr) - .parent() - .expect("shard") - .to_path_buf(); - drop(store); - - // Uppercase is deliberately rejected: on a case-folding filesystem accepting it - // would let one file answer to two index entries. - let upper = shard.join(hex::encode_upper(addressed("upper").0)); - std::fs::write(&upper, b"x").expect("write upper"); - std::fs::write(shard.join("not-a-chunk"), b"x").expect("write junk"); - std::fs::write(shard.join("deadbeef"), b"x").expect("write short"); - - let reopened = reopen(&dir).await; - assert_eq!(reopened.current_chunks().expect("count"), 1); - } - - #[tokio::test] - async fn a_chunk_filed_in_the_wrong_shard_is_not_indexed() { - let (store, dir) = test_store().await; - let (addr, content) = addressed("misfiled"); - store.put(&addr, &content).await.expect("put"); - drop(store); - - // Move it one shard over: the read path would never find it there, so indexing - // it would make the store advertise a key it cannot serve. - let correct = dir - .path() - .join(CHUNKS_DIR_NAME) - .join(shard_name(&addr)) - .join(hex::encode(addr)); - let wrong_shard_index = (shard_index(&addr) + 1) % SHARD_COUNT; - let wrong_dir = dir - .path() - .join(CHUNKS_DIR_NAME) - .join(format!("{wrong_shard_index:02x}")); - std::fs::create_dir_all(&wrong_dir).expect("mkdir"); - std::fs::rename(&correct, wrong_dir.join(hex::encode(addr))).expect("misfile"); - - let reopened = reopen(&dir).await; - assert_eq!(reopened.current_chunks().expect("count"), 0); - assert!(!reopened.exists(&addr).expect("exists")); - } - - #[tokio::test] - async fn the_layout_marker_is_written_once_and_checked_on_reopen() { - let (store, dir) = test_store().await; - drop(store); - - let marker = dir.path().join(CHUNKS_DIR_NAME).join(LAYOUT_FILE_NAME); - let layout: StoreLayout = - serde_json::from_slice(&std::fs::read(&marker).expect("read marker")) - .expect("parse marker"); - assert_eq!(layout, StoreLayout::default()); - - // A store written by a future build must be refused, not misread. - let future = StoreLayout { - schema: LAYOUT_SCHEMA + 1, - ..StoreLayout::default() - }; - std::fs::write(&marker, serde_json::to_vec(&future).expect("encode")).expect("write"); - let err = FileStore::new(FileStoreConfig { - root_dir: dir.path().to_path_buf(), - verify_on_read: true, - disk_reserve: 0, - }) - .await - .expect_err("must refuse a newer layout"); - assert!(format!("{err}").contains("newer than this build"), "{err}"); - } - - #[tokio::test] - async fn an_unknown_shard_scheme_is_refused() { - let (store, dir) = test_store().await; - drop(store); - let marker = dir.path().join(CHUNKS_DIR_NAME).join(LAYOUT_FILE_NAME); - let other = StoreLayout { - scheme: "prefix-hex".to_string(), - ..StoreLayout::default() - }; - std::fs::write(&marker, serde_json::to_vec(&other).expect("encode")).expect("write"); - let err = FileStore::new(FileStoreConfig { - root_dir: dir.path().to_path_buf(), - verify_on_read: true, - disk_reserve: 0, - }) - .await - .expect_err("must refuse an unknown scheme"); - assert!(format!("{err}").contains("shard scheme"), "{err}"); - } - - #[tokio::test] - async fn writes_are_refused_when_the_disk_reserve_cannot_be_met() { - let dir = TempDir::new().expect("temp dir"); - let store = FileStore::new(FileStoreConfig { - root_dir: dir.path().to_path_buf(), - verify_on_read: true, - disk_reserve: u64::MAX / 2, - }) - .await - .expect("open store"); - - let (addr, content) = addressed("full"); - let err = store.put(&addr, &content).await.expect_err("must refuse"); - assert!( - format!("{err}").contains("Insufficient disk space"), - "{err}" - ); - assert!(store.check_capacity().is_err()); - } - - #[tokio::test] - async fn capacity_is_size_aware() { - // Wide enough that a test running alongside this one cannot move the answer. - const MARGIN: u64 = 512 * 1024 * 1024; - - let dir = TempDir::new().expect("temp dir"); - let available = fs2::available_space(dir.path()).expect("free space"); - // A reserve that leaves room for a small write but not a huge one. This is the - // whole reason the predicate takes a size: free bytes alone stopped being a - // sufficient answer once chunks became files. - let store = FileStore::new(FileStoreConfig { - root_dir: dir.path().to_path_buf(), - verify_on_read: true, - disk_reserve: available.saturating_sub(MARGIN), - }) - .await - .expect("open store"); - - assert!(store.check_capacity_for(1024).is_ok()); - assert!(store.check_capacity_for(4 * MARGIN).is_err()); - } - - #[test] - fn suffix_shards_stay_uniform_for_a_close_group_of_keys() { - // The real distribution: a node holds keys it is closest to, so they share a - // long leading prefix with its own ID. Sharding on that prefix collapses to one - // directory. The trailing byte is untouched by close-group membership. - let mut prefix_dirs = HashSet::new(); - let mut suffix_dirs = HashSet::new(); - for i in 0u32..4096 { - let mut key = [0u8; XORNAME_LEN]; - // 20 shared leading bits, as a ~1M-node network would impose. - let tail = crate::client::compute_address(&i.to_le_bytes()); - key.copy_from_slice(&tail); - if let Some(b) = key.first_mut() { - *b = 0xab; - } - if let Some(b) = key.get_mut(1) { - *b = 0xcd; - } - if let Some(b) = key.get_mut(2) { - *b &= 0x0f; - } - prefix_dirs.insert(key.first().copied().unwrap_or(0)); - suffix_dirs.insert(shard_index(&key)); - } - assert_eq!( - prefix_dirs.len(), - 1, - "prefix sharding collapses for a node's own holdings" - ); - assert!( - suffix_dirs.len() > 250, - "suffix sharding must stay uniform, got {} of 256 directories", - suffix_dirs.len() - ); - } - - #[test] - fn no_chunk_filename_can_spell_a_reserved_windows_device_name() { - // Hex has no `n`, `u`, `x`, `p`, `r`, `l`, `t`, `o` or `s`, so `CON`, `NUL`, - // `AUX`, `PRN`, `COM1` and `LPT1` are all unspellable at any length. This is why - // the encoding is hex and not base32 or base64url. - for reserved in ["con", "prn", "aux", "nul", "com1", "com9", "lpt1", "lpt9"] { - assert!( - !is_lower_hex(reserved), - "{reserved} must not be a valid chunk or shard name" - ); - } - } - - #[test] - fn only_full_length_lowercase_hex_decodes_to_an_address() { - // 0xab so the hex form actually contains letters, which is where case matters. - assert!(decode_chunk_name(&hex::encode([0xabu8; XORNAME_LEN])).is_some()); - assert!(decode_chunk_name(&hex::encode_upper([0xabu8; XORNAME_LEN])).is_none()); - assert!(decode_chunk_name("deadbeef").is_none()); - assert!(decode_chunk_name("").is_none()); - assert!(decode_chunk_name(&"g".repeat(CHUNK_NAME_LEN)).is_none()); - } - - #[tokio::test] - async fn repair_replaces_bad_bytes_without_the_file_ever_being_absent() { - let (store, _dir) = test_store().await; - let (addr, content) = addressed("repairable"); - store.put(&addr, &content).await.expect("put"); - let path = store.chunk_path(&addr); - - std::fs::write(&path, b"rotted").expect("corrupt"); - store.repair(&addr, &content).await.expect("repair"); - - assert_eq!( - store.get(&addr).await.expect("get").expect("present"), - content - ); - assert!(store.exists(&addr).expect("exists")); - } - - #[tokio::test] - async fn a_repair_with_the_wrong_bytes_is_refused_and_changes_nothing() { - let (store, _dir) = test_store().await; - let (addr, content) = addressed("guarded"); - store.put(&addr, &content).await.expect("put"); - let path = store.chunk_path(&addr); - - // The whole point of repairing in place is that a failure must leave the old file - // where it was. Deleting first and writing after would open a window whose only - // surviving copy is the one the caller is about to destroy. - let err = store - .repair(&addr, b"not this chunk") - .await - .expect_err("must refuse"); - assert!(format!("{err}").contains("Refusing to repair"), "{err}"); - assert!( - path.exists(), - "the existing file must survive a refused repair" - ); - assert_eq!( - store.get(&addr).await.expect("get").expect("present"), - content - ); - } - - #[tokio::test] - async fn a_chunk_can_be_deleted_and_stored_again() { - let (store, _dir) = test_store().await; - let (addr, content) = addressed("cycle"); - - assert!(store.put(&addr, &content).await.expect("put")); - assert!(store.delete(&addr).await.expect("delete")); - assert!( - store.put(&addr, &content).await.expect("re-put"), - "a re-stored chunk is new again" - ); - assert_eq!( - store.get(&addr).await.expect("get").expect("present"), - content - ); - } - - /// Write a chunk file straight into its shard, the way an existing store already - /// contains thousands of them. Bypasses the write path deliberately: this exercises - /// the startup scan, not `put`. - fn plant(chunks_dir: &Path, key: &XorName) { - let dir = chunks_dir.join(shard_name(key)); - std::fs::create_dir_all(&dir).expect("mkdir"); - std::fs::write(dir.join(hex::encode(key)), key).expect("plant"); - } - - #[tokio::test] - async fn a_populated_and_churned_store_scans_correctly_at_scale() { - // Every shard populated, then aged the way a long-lived node ages: some keys - // deleted, others added in their place, so the directories carry holes rather - // than being freshly written. APFS enumeration is known to degrade with churn - // rather than with size, so a fresh corpus is not a realistic one. - const PLANTED: u32 = 20_000; - const CHURN: u32 = 1_000; - - let dir = TempDir::new().expect("temp dir"); - let chunks_dir = dir.path().join(CHUNKS_DIR_NAME); - std::fs::create_dir_all(&chunks_dir).expect("mkdir"); - - let mut expected: Vec = Vec::new(); - for i in 0..PLANTED { - let key = crate::client::compute_address(&i.to_le_bytes()); - plant(&chunks_dir, &key); - expected.push(key); - } - for i in 0..CHURN { - let key = crate::client::compute_address(&i.to_le_bytes()); - std::fs::remove_file(chunks_dir.join(shard_name(&key)).join(hex::encode(key))) - .expect("churn out"); - let replacement = crate::client::compute_address(&(PLANTED + i).to_le_bytes()); - plant(&chunks_dir, &replacement); - } - expected.retain(|k| chunks_dir.join(shard_name(k)).join(hex::encode(k)).exists()); - for i in 0..CHURN { - expected.push(crate::client::compute_address(&(PLANTED + i).to_le_bytes())); - } - expected.sort_unstable(); - expected.dedup(); - - let started = std::time::Instant::now(); - let store = reopen(&dir).await; - let scan = started.elapsed(); - - assert_eq!( - store.current_chunks().expect("count"), - expected.len() as u64 - ); - assert_eq!(store.all_keys().await.expect("all_keys"), expected); - - // Every shard should be in use at this size: 20,000 keys over 256 directories is - // about 78 each, and the last byte of a BLAKE3 output is uniform. - let occupied = std::fs::read_dir(&chunks_dir) - .expect("read store root") - .filter_map(std::result::Result::ok) - .filter(|e| e.file_name().to_str().is_some_and(|n| n.len() == 2)) - .count(); - assert_eq!(occupied, SHARD_COUNT, "the suffix must reach every shard"); - - println!( - "scan of {} keys across {SHARD_COUNT} shards took {scan:?}", - expected.len() - ); - } - - #[tokio::test] - async fn wait_idle_returns_once_writes_have_drained() { - let (store, _dir) = test_store().await; - let store = Arc::new(store); - for i in 0..32 { - let store = Arc::clone(&store); - let (addr, content) = addressed(&format!("drain-{i}")); - tokio::spawn(async move { store.put(&addr, &content).await }); - } - // Not a synchronisation point for tasks that have not been spawned yet, but it - // must not hang and it must leave the store usable. - store.wait_idle().await; - let (addr, content) = addressed("after-drain"); - assert!(store.put(&addr, &content).await.expect("put after drain")); - } -} diff --git a/src/storage/handler.rs b/src/storage/handler.rs index bd006817..b0338635 100644 --- a/src/storage/handler.rs +++ b/src/storage/handler.rs @@ -1,7 +1,7 @@ //! ANT protocol handler for autonomi protocol messages. //! //! This handler processes chunk PUT/GET requests with optional payment verification, -//! storing chunks to LMDB and using the DHT for network-wide retrieval. +//! storing chunks on disk and using the DHT for network-wide retrieval. //! //! # Architecture //! @@ -282,10 +282,10 @@ fn settlement_gate(client_settlement_version: u32, path: &str) -> Option, /// Payment verifier for checking payments. payment_verifier: Arc, @@ -305,7 +305,7 @@ impl AntProtocol { /// /// # Arguments /// - /// * `storage` - LMDB storage for chunk persistence + /// * `storage` - the chunk store /// * `payment_verifier` - Payment verifier for validating payments /// * `quote_generator` - Quote generator for creating storage quotes #[must_use] @@ -363,7 +363,7 @@ impl AntProtocol { CHUNK_PROTOCOL_ID } - /// Get a reference to the underlying LMDB storage. + /// Get a reference to the underlying chunk store. #[must_use] pub fn storage(&self) -> Arc { Arc::clone(&self.storage) @@ -764,13 +764,13 @@ impl AntProtocol { /// actually holds. /// /// The quote price is driven by `QuoteGenerator::records_stored()`. Reading - /// the live LMDB entry count (an O(1) B-tree page-header read) right before + /// the live chunk count right before /// pricing makes the metric deletion-aware: any chunk removed by /// [`ChunkStore::delete`] or by the replication prune pass is reflected /// immediately, with no risk of missing a delete path. /// /// On a storage read error — or a count that does not fit `usize` — the - /// previous metric value is left untouched so a transient LMDB error never + /// previous metric value is left untouched so a transient read error never /// disrupts quote generation. fn resync_quote_metric(&self) { match self.storage.current_chunks() { @@ -1332,7 +1332,7 @@ mod tests { /// "Full" now means both halves of the predicate: the volume is below the /// reserve **and** the store has no reusable space. A freshly created store /// has no freed pages, so both hold and the pre-check short-circuits, as it - /// always did. The companion cases in `storage::lmdb::tests` cover the half + /// always did. The companion cases in `storage::chunk_store::tests` cover the half /// that changed, where pruning has left reusable pages and the node must be /// admitted rather than refused on `statvfs` alone. /// diff --git a/src/storage/legacy_artifacts.rs b/src/storage/legacy_artifacts.rs new file mode 100644 index 00000000..44b98fef --- /dev/null +++ b/src/storage/legacy_artifacts.rs @@ -0,0 +1,918 @@ +//! What a node does when it finds the old chunk store still on disk. +//! +//! Chunks used to live in an LMDB environment at `{root}/chunks.mdb`. The previous release +//! copied them into a file per chunk and deleted that environment; this build has no code +//! that can read one. So a node starting with something still there needs an answer, and +//! there are three plausible ones. Two of them are wrong. +//! +//! **Refusing to start is wrong.** It was the first answer, and the reasoning was not silly: +//! those chunks are unreachable, so the node serves less than its published commitment +//! claims, and it spends the answerability window failing commitment-bound audits for keys +//! it cannot read. But a node that refuses serves *nothing* — not the chunks it cannot read, +//! and not the far larger number it migrated perfectly well. Nor can it be recovered: +//! `build_upgrade_monitor` is called unconditionally and `UpgradeConfig` has no field that +//! switches it off, so a node put back on the previous release is dragged forward again +//! within the hour, and on the deployed unit it is a ten-second restart loop until a person +//! intervenes. A few hours of trust penalty avoided, paid for with the whole node, +//! indefinitely. +//! +//! **Deleting whatever is there is also wrong**, and more obviously so once written down. +//! The upgrade monitor picks the newest eligible release rather than the next one, so a node +//! that was offline through the previous release arrives here with every chunk it owns in +//! that environment and nothing in the file store. Deleting it destroys data that may have +//! no other copy, to reclaim disk. +//! +//! So: **start, and remove only what is provably finished with.** The previous release wrote +//! a `RETIRED` mark inside the directory before it deleted anything, so a directory carrying +//! that mark is one whose retirement gates were all met and is pure cost. So is an empty one, +//! which is what a cleanup interrupted between emptying a tombstone and removing it leaves. +//! Those go, and their disk comes back. Anything else stays exactly where it is, and is +//! named once so an operator can decide. +//! +//! "Gates were met" is not "its contents are in the file store", and the difference is load +//! bearing. Retirement cleared a directory on two grounds: every chunk the node KEPT was copied +//! into the file store and re-hashed there, and every chunk it SHED was proven held by its close +//! group — all but one answering a possession challenge — and then deliberately not copied. So a +//! marked directory can legitimately hold bytes that are in no file store on this node. Removing +//! them is finishing what the previous release had already started; the safety argument for them +//! is the close group's proofs, not a local copy. +//! +//! **What may be deleted is decided by [`migration_signal::classify`], not by a second +//! reading of the same directory.** The previous release put each node's answer on the wire +//! so a fleet could be seen to have finished, and this release is published on the strength +//! of that count. If the cleanup had its own notion of which directories are finished with, +//! the two could drift, and a node could delete a directory it was still reporting as +//! unfinished, or report `files` while keeping one. They are one function, and the names +//! they match are one set of constants. +//! +//! Two things are never done, both because a name is not evidence of what is behind it. A +//! link is neither followed nor unlinked: what is behind it is on storage this node does not +//! own. And only the exact names the previous release created are considered at all. + +use std::path::{Path, PathBuf}; + +use crate::logging::{info, warn}; +use crate::storage::migration_signal::{classify, legacy_directories, Leftover, RETIRED_MARKER}; + +/// Remove what the storage migration finished with, and start either way. +/// +/// Returns nothing and fails at nothing. **This is what never vetoes a start** — it is not a +/// claim that every node starts, which is not this module's to make: the file store is built +/// before this runs and a store that cannot open still stops the node. What is ruled out is +/// a node kept from running by what it found left over, which is what the first draft of this +/// release did. +/// +/// Called once the file store has opened, and not before. "Finished with" means the chunks +/// are in the file store, which is only true if the file store is there to hold them: running +/// this earlier put the deletion in front of a constructor that can still fail, and a node +/// that lost both stores that way had nothing to go back to. A node that never opens a store +/// at all does not call this. +/// +/// This runs after the user agent that carries this node's migration state to every peer has +/// already been fixed, and that turns out not to matter: everything removed here is a +/// leftover the signal already calls finished with, because carrying the mark or being empty +/// is exactly what makes it removable and exactly what makes it harmless. The node announces +/// `files` either way, whether the deletion has finished or not yet started. +pub fn clean_up(root_dir: &Path) { + // A root that cannot be listed hides every tombstone under it. The previous release's + // own reporting calls that `unknown` rather than `files`, and the matching answer here is + // to remove nothing and say so: an unlistable root is not evidence that there is nothing + // to keep. Said out loud because an earlier version returned silently, which left the + // decision record promising a warning that no code emitted. + let Ok(leftovers) = legacy_directories(root_dir) else { + warn!( + migration_event = "legacy_store_left", + "{} could not be listed, so nothing left over from the storage migration was \ + removed from it. Any leftover is still costing disk.", + root_dir.display() + ); + return; + }; + + let mut finished_with = Vec::new(); + for dir in leftovers { + match classify(&dir) { + // Its chunks are in the file store and the previous release simply did not + // finish deleting it, or there is nothing in it at all. Either way it holds + // nothing, so removing it cannot lose anything. + Leftover::Harmless => finished_with.push(dir), + Leftover::Holding | Leftover::Unreadable => warn!( + migration_event = "legacy_store_left", + "{} is left over from the storage migration and this build will not remove \ + it: {}. It is costing disk until it is dealt with by hand.", + dir.display(), + why_it_is_kept(&dir) + ), + } + } + + if !finished_with.is_empty() { + remove_all(finished_with); + } +} + +/// Why one directory is being kept, for the operator who has to decide what to do with it. +/// +/// Prose only. The decision was already made by `classify`; this re-reads the directory +/// solely to say which of its reasons applied. If the disk changes underneath the two, the +/// cost is a warning that names the wrong reason, never a directory deleted that should not +/// have been. +/// +/// Its only caller is inside a `warn!`, which compiles to nothing without the `logging` +/// feature, so off that feature this has no caller at all. Same treatment as the signal's own +/// token helpers rather than a `#[cfg]`, which would take the function out of the build the +/// tests run in. +#[cfg_attr(not(feature = "logging"), allow(dead_code))] +fn why_it_is_kept(dir: &Path) -> &'static str { + let Ok(meta) = std::fs::symlink_metadata(dir) else { + return "it cannot be examined"; + }; + if meta.file_type().is_symlink() { + return "it is a link to storage this node does not own"; + } + if !meta.is_dir() { + return "it is not a directory"; + } + match std::fs::symlink_metadata(dir.join(RETIRED_MARKER)) { + Ok(meta) if meta.is_file() => "it carries the mark that says it was finished with", + Ok(_) => { + "something that is not the storage migration's mark is using the name the mark \ + would have, so this node cannot tell whether it was finished with" + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + "it has chunks in it that were never copied into the file store, and this build \ + cannot read them. Nothing here will delete them, so they are not lost. Whether \ + this node's close group still needs them is the question to answer before \ + removing it by hand" + } + Err(_) => "whether it was finished with cannot be established", + } +} + +/// Delete every directory that provably holds no chunks, one after another. +/// +/// **One thread, in sequence.** The names this release recognises are the live directory, the +/// unnumbered tombstone and sixty-four numbered ones, so a root that has been through enough +/// restore cycles can present sixty-six at once. A thread each would put sixty-six concurrent +/// recursive deletions on the disk that is also serving chunks, at the moment a node is +/// starting up. They are pure disk work with nothing waiting on them, so doing them in turn +/// costs nothing that matters and bounds what this can do to a node's I/O. +/// +/// In the background and in place. Nothing has to be got out of the way first: these +/// directories hold nothing, and this build has no code that would read them if they did. Not +/// renaming also means no name to allocate, which is what an earlier version of this could +/// run out of and wedge itself on. +fn remove_all(dirs: Vec) { + let spawned = std::thread::Builder::new() + .name("legacy-store-cleanup".into()) + .spawn(move || { + for dir in dirs { + match delete_mark_last(&dir) { + // Said only once the deletion has finished. Announcing the space before + // it is back is how an operator comes to trust a number that is wrong for + // the next several minutes. + Ok(()) => info!( + migration_event = "space_returned", + "Removed {}, which the storage migration had finished with, and \ + returned its space.", + dir.display() + ), + Err(e) => warn!( + migration_event = "legacy_store_left", + "{} was finished with by the storage migration but could not be \ + removed ({e}). It is costing disk. The next start tries again.", + dir.display() + ), + } + } + }); + if let Err(e) = spawned { + // Naming what did not happen, not just that something did not. An earlier version + // said only that a thread could not start, which tells an operator nothing about + // which disk stayed full. + warn!( + migration_event = "legacy_store_left", + "Could not start the storage migration's cleanup thread ({e}), so nothing it had \ + finished with was removed. Those directories are still costing disk and the next \ + start tries again." + ); + } +} + +/// Empty a directory, then remove its mark, then remove the directory. +/// +/// The order is the whole of it. `remove_dir_all` gives no promise about which entry it +/// unlinks first, and the mark is the only thing that says this directory was finished with: +/// if it goes before the chunks do and the process stops there, the next start finds an +/// unmarked directory with data in it, decides it might be an unmigrated store, and keeps it +/// forever. It is not one, but nothing on disk says so any more, and the node then reports +/// itself as unfinished to the whole network for as long as it lives. +/// +/// Done this way there is no such moment. At every point either the mark is still there, and +/// the next start resumes, or the directory is empty or gone, which is also finished with. +/// +/// **On Unix nothing here names a path twice.** An earlier version checked +/// `symlink_metadata(dir)` and then re-opened the same path with `read_dir`, which follows a +/// link. Anything that could replace the directory between the two — a local actor with write +/// access to the node's data directory — could point it at a target elsewhere on the disk and +/// have this process delete that target's contents instead. The node can reach more of the +/// filesystem than such an actor can, so that is not one more way to lose data inside the data +/// directory: it is a way to reach outside it. The directory is now opened once, `O_NOFOLLOW` +/// and `O_DIRECTORY`, and every unlink is made against that handle, so the answer that +/// authorises the deletion and the deletion itself are about the same inode by construction +/// rather than by hope. +/// +/// The directory itself still goes by path, and that is safe on its own terms: `rmdir` refuses +/// a symlink, so a swapped name fails the call rather than following it. +/// +/// Off Unix there is no `unlinkat`, so the path-based version stays. Its exposure is the same +/// as the previous release's deleter had, and it is written down in ADR-0015 rather than +/// implied. +fn delete_mark_last(dir: &Path) -> std::io::Result<()> { + empty_but_for_the_mark(dir)?; + // Now, and only now, the thing that said it was safe to do any of the above. + remove_the_mark(dir)?; + // `rmdir`, not `unlink`: it refuses a symlink outright, so this one path cannot be + // followed anywhere even if the name were swapped between the emptying and here. + std::fs::remove_dir(dir) +} + +/// The gates, and then the contents, all against one directory handle. +#[cfg(unix)] +fn empty_but_for_the_mark(dir: &Path) -> std::io::Result<()> { + let dirfd = open_dir_nofollow(dir)?; + + // Asked again, here, on the thread that does the deleting, and asked of the handle. The + // classification that got this path happened on another thread and is already in the past. + // Opening with `O_NOFOLLOW | O_DIRECTORY` is what makes the re-ask sound: a link or a + // non-directory fails the open rather than being examined and then acted on separately. + let marked = matches!( + rustix::fs::statat(&dirfd, RETIRED_MARKER, rustix::fs::AtFlags::SYMLINK_NOFOLLOW), + Ok(stat) if stat.st_mode & rustix::fs::FileType::RegularFile.as_raw_mode() + == rustix::fs::FileType::RegularFile.as_raw_mode() + ); + if !marked && !is_empty_but_for_the_mark(&dirfd)? { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "it no longer carries the mark that said it was finished with, and it is not \ + empty either", + )); + } + empty_at(&dirfd, true) +} + +/// Open a directory, refusing a link or anything that is not one, on the handle. +#[cfg(unix)] +fn open_dir_nofollow(dir: &Path) -> std::io::Result { + use rustix::fs::{Mode, OFlags}; + + rustix::fs::open( + dir, + OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC, + Mode::empty(), + ) + .map_err(|e| { + // A link, or something that is not a directory at all, arrives here as ELOOP or + // ENOTDIR. Both mean the same thing to this caller and it is worth saying which. + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!( + "{} is no longer the directory that was found to be finished with ({e})", + dir.display() + ), + ) + }) +} + +/// Is there anything in here besides the mark? +#[cfg(unix)] +fn is_empty_but_for_the_mark(dirfd: &std::os::fd::OwnedFd) -> std::io::Result { + for name in read_names(dirfd)? { + if name.as_bytes() != RETIRED_MARKER.as_bytes() { + return Ok(false); + } + } + Ok(true) +} + +/// Unlink everything under `dirfd`, optionally sparing the mark, recursing on handles. +#[cfg(unix)] +fn empty_at(dirfd: &std::os::fd::OwnedFd, spare_the_mark: bool) -> std::io::Result<()> { + use rustix::fs::AtFlags; + + for name in read_names(dirfd)? { + if spare_the_mark && name.as_bytes() == RETIRED_MARKER.as_bytes() { + continue; + } + // Try the plain unlink first. A directory answers EISDIR (EPERM on some systems), and + // that answer is about the inode the handle names, so it cannot be redirected. Doing + // it this way round also means the common case — a file — costs one syscall. + match rustix::fs::unlinkat(dirfd, name.as_c_str(), AtFlags::empty()) { + // Gone already is the outcome this wanted. + Ok(()) | Err(rustix::io::Errno::NOENT) => continue, + // A directory: EISDIR on Linux, EPERM on the BSDs and macOS. Both answers are + // about the inode the handle names, so neither can have been redirected. + Err(rustix::io::Errno::ISDIR | rustix::io::Errno::PERM) => {} + Err(e) => return Err(e.into()), + } + // A subdirectory. Opened `O_NOFOLLOW` from this handle, emptied the same way, then + // removed by name against the handle that named it. + let child = rustix::fs::openat( + dirfd, + name.as_c_str(), + rustix::fs::OFlags::RDONLY + | rustix::fs::OFlags::DIRECTORY + | rustix::fs::OFlags::NOFOLLOW + | rustix::fs::OFlags::CLOEXEC, + rustix::fs::Mode::empty(), + )?; + empty_at(&child, false)?; + drop(child); + rustix::fs::unlinkat(dirfd, name.as_c_str(), AtFlags::REMOVEDIR)?; + } + Ok(()) +} + +/// Every name in a directory handle, without `.` and `..`. +/// +/// Collected rather than streamed: the unlinking below changes what a live iterator would see +/// next, and reading the whole list first is both simpler to reason about and small — these +/// directories hold a database file, a lock file and a mark. +#[cfg(unix)] +fn read_names(dirfd: &std::os::fd::OwnedFd) -> std::io::Result> { + let mut names = Vec::new(); + let dir = rustix::fs::Dir::read_from(dirfd)?; + for entry in dir { + let entry = entry?; + let name = entry.file_name(); + if name.to_bytes() == b"." || name.to_bytes() == b".." { + continue; + } + names.push(name.to_owned()); + } + Ok(names) +} + +/// The path-based version, for the platforms with no `unlinkat`. +#[cfg(not(unix))] +fn empty_but_for_the_mark(dir: &Path) -> std::io::Result<()> { + if !matches!(std::fs::symlink_metadata(dir), Ok(meta) if meta.is_dir()) { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "it is no longer the directory that was found to be finished with", + )); + } + let marked = matches!( + std::fs::symlink_metadata(dir.join(RETIRED_MARKER)), + Ok(meta) if meta.is_file() + ); + if !marked && std::fs::read_dir(dir)?.next().is_some() { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "it no longer carries the mark that said it was finished with, and it is not \ + empty either", + )); + } + for entry in std::fs::read_dir(dir)? { + let entry = entry?; + if entry.file_name() == RETIRED_MARKER { + continue; + } + if entry.file_type()?.is_dir() { + std::fs::remove_dir_all(entry.path())?; + } else { + std::fs::remove_file(entry.path())?; + } + } + Ok(()) +} + +/// Remove the mark, which is what said any of the above was allowed. +#[cfg(unix)] +fn remove_the_mark(dir: &Path) -> std::io::Result<()> { + let dirfd = open_dir_nofollow(dir)?; + match rustix::fs::unlinkat(&dirfd, RETIRED_MARKER, rustix::fs::AtFlags::empty()) { + // Already gone is success: something else finished this, or there was never a mark + // because the directory qualified by being empty. + Ok(()) | Err(rustix::io::Errno::NOENT) => Ok(()), + Err(e) => Err(e.into()), + } +} + +/// Remove the mark, which is what said any of the above was allowed. +#[cfg(not(unix))] +fn remove_the_mark(dir: &Path) -> std::io::Result<()> { + match std::fs::remove_file(dir.join(RETIRED_MARKER)) { + Ok(()) => Ok(()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(e), + } +} + +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +mod tests { + use super::*; + use crate::storage::migration_signal::{is_tombstone_name, LEGACY_ENV_DIR, MAX_TOMBSTONES}; + use tempfile::TempDir; + + /// The deletion runs on its own thread. + fn wait_gone(path: &Path) -> bool { + for _ in 0..200 { + if !path.exists() { + return true; + } + std::thread::sleep(std::time::Duration::from_millis(10)); + } + false + } + + fn settle() { + std::thread::sleep(std::time::Duration::from_millis(200)); + } + + fn env_with_chunks(root: &Path, name: &str) -> PathBuf { + let dir = root.join(name); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("data.mdb"), b"chunks").unwrap(); + dir + } + + fn mark_retired(dir: &Path) { + std::fs::write(dir.join(RETIRED_MARKER), b"retired").unwrap(); + } + + /// The property this module exists for: whatever is on disk, the node runs. + #[test] + fn nothing_here_can_stop_a_node_starting() { + let root = TempDir::new().unwrap(); + env_with_chunks(root.path(), LEGACY_ENV_DIR); + env_with_chunks(root.path(), "chunks.mdb.retired.2"); + std::fs::create_dir_all(root.path().join("chunks.mdb.retired")).unwrap(); + + // No Result to unwrap: there is no way for this to refuse. + clean_up(root.path()); + settle(); + } + + #[test] + fn a_root_that_does_not_exist_yet_is_fine() { + let root = TempDir::new().unwrap(); + clean_up(&root.path().join("not").join("created")); + } + + /// The one this release is for: a store the migration finished with but did not delete. + #[test] + fn a_store_the_migration_finished_with_is_removed() { + let root = TempDir::new().unwrap(); + let env = env_with_chunks(root.path(), LEGACY_ENV_DIR); + mark_retired(&env); + clean_up(root.path()); + assert!(wait_gone(&env)); + } + + #[test] + fn a_marked_tombstone_is_removed_too() { + let root = TempDir::new().unwrap(); + let tomb = env_with_chunks(root.path(), "chunks.mdb.retired.7"); + mark_retired(&tomb); + clean_up(root.path()); + assert!(wait_gone(&tomb)); + } + + #[test] + fn an_empty_leftover_is_removed() { + // What a cleanup interrupted between emptying a tombstone and removing it leaves. + let root = TempDir::new().unwrap(); + let tomb = root.path().join("chunks.mdb.retired"); + std::fs::create_dir_all(&tomb).unwrap(); + clean_up(root.path()); + assert!(wait_gone(&tomb)); + } + + /// A marked directory goes even though its bytes are in no file store on this node. + /// + /// This is the case the record used to describe wrongly, and the tests used to conceal by + /// never staging a file store at all: they showed that a mark triggers deletion, which + /// leaves a reader free to assume the deleted bytes had been copied somewhere first. + /// + /// They need not have been. Retirement cleared a directory on two grounds, and only one is + /// a local copy: chunks the node KEPT were copied into the file store and re-hashed there, + /// and chunks it SHED were proven held by its close group and then deliberately not copied + /// — the pre-retirement pass skips exactly those keys. So the state staged here, a marked + /// directory beside an empty file store, is one the previous release produces on purpose on + /// any node that was short of disk, and it is reachable through its own kill point between + /// writing the mark and finishing the delete. + /// + /// Removing it is right: the previous release was about to. What this pins is that the + /// deletion does NOT depend on a local copy existing, so nobody later "fixes" it by adding + /// a file-store check that would strand every shedding node's directory for ever. + #[test] + fn a_marked_store_goes_even_when_nothing_was_copied_into_this_node() { + let root = TempDir::new().unwrap(); + let env = env_with_chunks(root.path(), LEGACY_ENV_DIR); + mark_retired(&env); + + // No file store, and nothing in one. A node that shed everything it held looks exactly + // like this. + assert!(!root.path().join("chunks").exists()); + + clean_up(root.path()); + assert!( + wait_gone(&env), + "a marked directory must go whether or not this node kept a copy: the close \ + group's possession proofs are what cleared it, not a local file" + ); + } + + /// The one that would be data loss. + /// + /// A node that missed the previous release entirely arrives here with every chunk it + /// owns in that directory and nothing in the file store, and the upgrade monitor picks + /// the newest release rather than the next one, so this build is genuinely reachable + /// from that state. Unmarked is the whole difference from the test above: nothing + /// cleared this directory, so nothing here may remove it. + #[test] + fn a_store_that_was_never_migrated_is_left_exactly_where_it_is() { + let root = TempDir::new().unwrap(); + let env = env_with_chunks(root.path(), LEGACY_ENV_DIR); + clean_up(root.path()); + settle(); + assert!( + env.join("data.mdb").exists(), + "an unmarked store passed no retirement gate, and was deleted anyway" + ); + } + + #[test] + fn an_unmarked_tombstone_with_chunks_in_it_is_left_too() { + // A crash between the rename and the mark leaves an intact environment wearing a + // retired-looking name. What it is called is not evidence. + let root = TempDir::new().unwrap(); + let tomb = env_with_chunks(root.path(), "chunks.mdb.retired.3"); + clean_up(root.path()); + settle(); + assert!(tomb.join("data.mdb").exists()); + } + + /// A name is not evidence, and this decides what gets deleted. + #[test] + fn a_directory_that_only_looks_like_a_tombstone_is_left_alone() { + let root = TempDir::new().unwrap(); + let mine = env_with_chunks(root.path(), "chunks.mdb.retired-keep-this"); + mark_retired(&mine); + let also = env_with_chunks(root.path(), "chunks.mdb.backup"); + mark_retired(&also); + let high = env_with_chunks(root.path(), "chunks.mdb.retired.65"); + mark_retired(&high); + let padded = env_with_chunks(root.path(), "chunks.mdb.retired.007"); + mark_retired(&padded); + + clean_up(root.path()); + settle(); + for kept in [&mine, &also, &high, &padded] { + assert!( + kept.exists(), + "{} was deleted and nothing here created it", + kept.display() + ); + } + + assert!(is_tombstone_name("chunks.mdb.retired")); + assert!(is_tombstone_name("chunks.mdb.retired.1")); + assert!(is_tombstone_name("chunks.mdb.retired.64")); + assert!(!is_tombstone_name("chunks.mdb.retired.65")); + assert!(!is_tombstone_name("chunks.mdb.retired.0")); + assert!(!is_tombstone_name("chunks.mdb.retired.007")); + assert!(!is_tombstone_name("chunks.mdb.retired-keep-this")); + assert!(!is_tombstone_name("chunks.mdb.retired.")); + assert!(!is_tombstone_name("chunks.mdb.retired.x")); + } + + /// The name is reserved, but only a file R2 could have written is evidence. + #[test] + fn something_else_wearing_the_marks_name_is_not_the_mark() { + let root = TempDir::new().unwrap(); + let env = env_with_chunks(root.path(), LEGACY_ENV_DIR); + // A directory at the reserved name, which `try_exists` cannot tell from the file + // retirement writes. Believing it deletes an unmigrated store. + std::fs::create_dir_all(env.join(RETIRED_MARKER)).unwrap(); + + clean_up(root.path()); + settle(); + assert!( + env.join("data.mdb").exists(), + "a directory called RETIRED authorised deleting an unmigrated store" + ); + } + + /// The mark is removed last, so an interrupted deletion is still recognisable. + /// + /// `remove_dir_all` promises nothing about order. If the mark went first and the process + /// stopped there, the next start would find an unmarked directory with data in it, treat + /// it as a store that was never migrated, and keep it for good — and the node would + /// report itself unfinished to the whole network for as long as it lived. + #[test] + fn the_mark_outlives_everything_it_was_vouching_for() { + let root = TempDir::new().unwrap(); + let env = env_with_chunks(root.path(), LEGACY_ENV_DIR); + std::fs::write(env.join("lock.mdb"), b"lock").unwrap(); + mark_retired(&env); + + // The deletion, run inline so its intermediate states can be inspected rather than + // raced. + delete_mark_last(&env).unwrap(); + assert!(!env.exists()); + + // And the order it does it in: with the directory made unremovable, the mark has to + // still be there when the attempt fails. + let other = TempDir::new().unwrap(); + let env = env_with_chunks(other.path(), LEGACY_ENV_DIR); + mark_retired(&env); + let nested = env.join("sub"); + std::fs::create_dir_all(&nested).unwrap(); + std::fs::write(nested.join("data.mdb"), b"chunks").unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&nested, std::fs::Permissions::from_mode(0o500)).unwrap(); + let failed = delete_mark_last(&env); + std::fs::set_permissions(&nested, std::fs::Permissions::from_mode(0o755)).unwrap(); + assert!(failed.is_err(), "the deletion was supposed to fail here"); + assert!( + env.join(RETIRED_MARKER).exists(), + "the mark went before the chunks did, so the next start cannot tell this \ + directory was already finished with" + ); + } + } + + /// The deleting thread asks again rather than trusting a path it was handed. + /// + /// What is at a path is not a property of the path. The classification happened on + /// another thread and is already in the past, so between the two moments the directory + /// can be replaced with a link to somewhere else, and following it would delete through + /// it. Staged directly here because a real race is not reproducible in a test. + #[cfg(unix)] + #[test] + fn the_deleting_thread_refuses_a_path_that_is_no_longer_what_was_classified() { + let root = TempDir::new().unwrap(); + let elsewhere = env_with_chunks(root.path(), "elsewhere"); + // MARKED, and that is the whole point of this test. An unmarked target would be + // refused by the gates even by a deleter that happily followed the link, so the test + // would pass while proving nothing about following it. Marked, the target satisfies + // every gate: anything that reaches it through the link deletes it. + mark_retired(&elsewhere); + let swapped = root.path().join(LEGACY_ENV_DIR); + std::os::unix::fs::symlink(&elsewhere, &swapped).unwrap(); + + assert!( + delete_mark_last(&swapped).is_err(), + "the deletion followed a link that appeared after the classification" + ); + assert!( + elsewhere.join("data.mdb").exists(), + "the link was followed and the contents of somewhere else were deleted" + ); + assert!(elsewhere.join(RETIRED_MARKER).exists()); + + // And a directory that lost its mark in between: it might be a store nothing + // migrated, so it is not deleted on the strength of an answer given earlier. + let unmarked = env_with_chunks(root.path(), "chunks.mdb.retired.9"); + assert!(delete_mark_last(&unmarked).is_err()); + assert!(unmarked.join("data.mdb").exists()); + } + + /// A link never yields a directory handle, which is what leaves no window to win. + /// + /// The test above stages a link that is already there when the deletion starts, and the + /// path-based version caught that too, by asking `symlink_metadata` first. What it could + /// not catch is a link that arrives *between* that question and the `read_dir` that acted + /// on the answer: `read_dir` resolves the path again and follows what it finds. A local + /// actor with write access to the node's data directory could point that name at a target + /// elsewhere on the disk, and the node — which reaches more of the filesystem than the + /// actor does, and on plenty of installations runs as root — would empty the target + /// instead. Not one more way to lose data inside the data directory: a way out of it. + /// + /// That interleaving cannot be staged from a test without instrumenting the deleter, so + /// what is pinned here is the primitive that removes the window rather than the race + /// itself. The directory is opened once, and every unlink is made against that handle, so + /// if a link can never produce a handle then no unlink can ever be redirected through one. + /// This is the "can never produce a handle" half, and it is the half that a change would + /// silently undo: drop `O_NOFOLLOW` and the open below starts succeeding. + #[cfg(unix)] + #[test] + fn a_link_never_yields_a_directory_handle() { + let root = TempDir::new().unwrap(); + let real = env_with_chunks(root.path(), "real"); + + assert!( + open_dir_nofollow(&real).is_ok(), + "a directory that is what it says it is has to open, or nothing is ever cleaned up" + ); + + let link = root.path().join("link"); + std::os::unix::fs::symlink(&real, &link).unwrap(); + assert!( + open_dir_nofollow(&link).is_err(), + "a link produced a handle, so every unlink made against it lands on the target" + ); + + // The other thing that must never be mistaken for a store. A FIFO would block a + // plain open forever; refusing on the handle refuses it without opening it. + let not_a_dir = root.path().join("file"); + std::fs::write(¬_a_dir, b"x").unwrap(); + assert!(open_dir_nofollow(¬_a_dir).is_err()); + } + + /// The cleanup removes exactly what the fleet signal calls finished with, and nothing else. + /// + /// This is the property that lets the previous release's count authorise this one. That + /// count is each node reporting `files` when nothing under its root is `Leftover::Holding` + /// or `Leftover::Unreadable`. If this release deleted anything the signal would not have + /// called harmless, a node could destroy a directory it was still reporting as unfinished; + /// if it kept something the signal called harmless, a node would report `files` and go on + /// paying for the disk for ever. + /// + /// They cannot disagree today because there is one `classify`. This is what fails if + /// somebody gives the cleanup its own again: every shape below is checked both ways round, + /// so a classifier that is wrong in either direction shows up here rather than on a fleet. + #[test] + fn the_cleanup_removes_exactly_what_the_signal_calls_finished_with() { + let root = TempDir::new().unwrap(); + let base = root.path(); + + // Every shape a real root can present, harmless and not, with the near misses that + // are not this release's to touch at all. + let marked_live = env_with_chunks(base, LEGACY_ENV_DIR); + mark_retired(&marked_live); + let marked_tomb = env_with_chunks(base, "chunks.mdb.retired.5"); + mark_retired(&marked_tomb); + let empty_tomb = base.join("chunks.mdb.retired"); + std::fs::create_dir_all(&empty_tomb).unwrap(); + let unmarked_tomb = env_with_chunks(base, "chunks.mdb.retired.6"); + let fake_mark = env_with_chunks(base, "chunks.mdb.retired.7"); + std::fs::create_dir_all(fake_mark.join(RETIRED_MARKER)).unwrap(); + let near_miss = env_with_chunks(base, "chunks.mdb.retired.65"); + mark_retired(&near_miss); + let operators = env_with_chunks(base, "chunks.mdb.retired-keep-this"); + mark_retired(&operators); + + // Not a directory at all, wearing a name in the set. The marker probe cannot even be + // asked of it, so it classifies unreadable and must be kept. + let not_a_dir = base.join("chunks.mdb.retired.8"); + std::fs::write(¬_a_dir, b"not a store").unwrap(); + + // A link wearing a name in the set. What is behind it is somebody else's, and neither + // following it nor unlinking it is this build's decision. + #[cfg(unix)] + let linked = { + let elsewhere = env_with_chunks(base, "elsewhere-for-the-link"); + mark_retired(&elsewhere); + let link = base.join("chunks.mdb.retired.9"); + std::os::unix::fs::symlink(&elsewhere, &link).unwrap(); + (link, elsewhere) + }; + + // What the signal says about each, before anything is removed. Every classification + // the enum can produce is represented: Harmless three ways, Holding two, Unreadable + // two. + // The `mut` is used only where the link below exists, which is not everywhere. + #[cfg_attr(not(unix), allow(unused_mut))] + let mut considered = vec![ + &marked_live, + &marked_tomb, + &empty_tomb, + &unmarked_tomb, + &fake_mark, + ¬_a_dir, + ]; + #[cfg(unix)] + considered.push(&linked.0); + let verdicts: Vec<_> = considered.iter().map(|dir| (*dir, classify(dir))).collect(); + + clean_up(base); + settle(); + + for (dir, verdict) in verdicts { + let gone = !dir.exists(); + assert_eq!( + gone, + verdict == Leftover::Harmless, + "{} was classified {verdict:?} and {} removed", + dir.display(), + if gone { "was" } else { "was not" } + ); + } + + // And the names outside the set are not classified at all, so they are never even + // considered for deletion. + for untouched in [&near_miss, &operators] { + assert!( + untouched.join("data.mdb").exists(), + "{} is not a name retirement can have created", + untouched.display() + ); + } + + // A link is kept as a link, and what it points at keeps its data: following it would + // delete storage this node does not own, unlinking it would throw away the only record + // of where that data went. + #[cfg(unix)] + { + let (link, elsewhere) = &linked; + assert!( + std::fs::symlink_metadata(link).is_ok(), + "the link was removed" + ); + assert!( + elsewhere.join("data.mdb").exists(), + "the link was followed and somebody else's data was deleted" + ); + } + } + + /// Sixty-six leftovers are removed one after another, on one thread. + /// + /// The accepted names are the live directory, the unnumbered tombstone and sixty-four + /// numbered ones. A thread each would put sixty-six recursive deletions on the disk that + /// is also serving chunks, at the moment the node is starting. + #[test] + fn every_leftover_a_root_can_hold_is_removed_without_a_thread_each() { + let root = TempDir::new().unwrap(); + let base = root.path(); + + let mut staged = vec![env_with_chunks(base, LEGACY_ENV_DIR)]; + staged.push(env_with_chunks(base, "chunks.mdb.retired")); + for n in 1..=MAX_TOMBSTONES { + staged.push(env_with_chunks(base, &format!("chunks.mdb.retired.{n}"))); + } + assert_eq!(staged.len(), 66, "the accepted namespace is this big"); + for dir in &staged { + mark_retired(dir); + } + + let before = std::thread::available_parallelism().is_ok(); + clean_up(base); + assert!(before, "sanity: the platform reports its parallelism"); + + for dir in &staged { + assert!( + wait_gone(dir), + "{} was finished with and is still there", + dir.display() + ); + } + } + + /// A root that cannot be listed removes nothing, and says so. + /// + /// An unlistable root hides every tombstone under it, so it is not evidence that there is + /// nothing to keep. An earlier version returned silently here, which left the decision + /// record promising a warning that no code emitted. + #[cfg(unix)] + #[test] + fn a_root_that_cannot_be_listed_removes_nothing() { + use std::os::unix::fs::PermissionsExt; + + let root = TempDir::new().unwrap(); + let base = root.path().join("locked"); + std::fs::create_dir_all(&base).unwrap(); + let env = env_with_chunks(&base, LEGACY_ENV_DIR); + mark_retired(&env); + + std::fs::set_permissions(&base, std::fs::Permissions::from_mode(0o000)).unwrap(); + clean_up(&base); + settle(); + std::fs::set_permissions(&base, std::fs::Permissions::from_mode(0o755)).unwrap(); + + assert!( + env.join("data.mdb").exists(), + "a root that could not be listed had something removed from it anyway" + ); + } + + /// What a link points at belongs to somebody else, and is never followed or removed. + #[cfg(unix)] + #[test] + fn a_linked_environment_is_left_exactly_as_it_is() { + let root = TempDir::new().unwrap(); + let elsewhere = env_with_chunks(root.path(), "elsewhere"); + mark_retired(&elsewhere); + let link = root.path().join(LEGACY_ENV_DIR); + std::os::unix::fs::symlink(&elsewhere, &link).unwrap(); + + clean_up(root.path()); + settle(); + + assert!( + std::fs::symlink_metadata(&link).is_ok(), + "the link itself was removed, which throws away the only record of where the \ + data went" + ); + assert!( + elsewhere.join("data.mdb").exists(), + "the link was followed and somebody else's data was deleted" + ); + } +} diff --git a/src/storage/lmdb.rs b/src/storage/lmdb.rs deleted file mode 100644 index 82b97cca..00000000 --- a/src/storage/lmdb.rs +++ /dev/null @@ -1,2446 +0,0 @@ -//! Content-addressed LMDB storage for chunks. -//! -//! Provides persistent storage for chunks using LMDB (via heed) for -//! memory-mapped, zero-copy reads with ACID transactions. -//! -//! ```text -//! {root}/chunks.mdb/ -- LMDB environment directory -//! ``` - -use crate::ant_protocol::{XorName, MAX_CHUNK_SIZE}; -use crate::error::{Error, Result}; -use crate::logging::{debug, info, trace, warn}; -use heed::types::Bytes; -use heed::{Database, Env, EnvOpenOptions, MdbError}; -use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::sync::Arc; -use std::time::Instant; -use tokio::task::spawn_blocking; -use tokio_util::task::TaskTracker; - -use crate::ant_protocol::XORNAME_LEN; -use crate::storage::StorageStats; - -use crate::storage::{GIB, MIB}; - -/// Default minimum free disk space to preserve on the storage partition. -const DEFAULT_DISK_RESERVE: u64 = 500 * MIB; - -/// Convert a byte count to GiB for human-readable log messages. -#[allow(clippy::cast_precision_loss)] // display only — sub-byte precision is irrelevant -fn bytes_to_gib(bytes: u64) -> f64 { - bytes as f64 / GIB as f64 -} - -/// Absolute minimum LMDB map size. -/// -/// Even on a nearly-full disk the database must be able to open. -/// Set to 256 MiB — enough for millions of LMDB pages. -const MIN_MAP_SIZE: usize = 256 * 1024 * 1024; - -/// Maximum head-room (beyond the current data footprint) to reserve for the -/// LMDB map **on Windows**. -/// -/// On Windows a node's committed / private memory scales with the *mapped* -/// size rather than with the data actually stored. Measured on a live node: -/// ~7.4 GB of extra commit for a ~3.7 TiB map, versus ~181 MB for a ~55 GiB -/// map — roughly 0.2% of the mapped size, resident and attributed to *no* -/// user-space allocation. That size-proportional cost is consistent with -/// kernel page tables / section metadata for the mapping; the exact kernel -/// structure was inferred from the scaling rather than measured directly, but -/// the size-proportional *effect* is what this cap targets. Linux keeps the -/// mapping sparse, so a disk-sized map is nearly free there — the overhead is -/// Windows-specific, which is why it only surfaced in Windows reports. -/// -/// Sizing the map to the whole disk therefore costs ~0.2% of *free disk* per -/// node, multiplied by every node sharing the host (e.g. a 10 TiB partition ≈ -/// 20 GiB). We instead cap the head-room on Windows and lean on -/// `LmdbStorage::try_resize` to extend the map on demand as data accumulates, -/// keeping the overhead proportional to *stored data* rather than *disk -/// capacity*. At 32 GiB the extra commit is ~100 MB, and a resize happens at -/// most once per 32 GiB written. -#[cfg(windows)] -const WINDOWS_MAP_HEADROOM: u64 = 32 * GIB; - -/// How often to re-query available disk space (in seconds). -/// -/// Between checks the cached result is trusted. Disk space changes slowly -/// relative to chunk-write throughput, so a multi-second window is safe. -const DISK_CHECK_INTERVAL_SECS: u64 = 5; - -/// Ceiling raise offered to a single *delete* that cannot copy-on-write inside -/// the pinned map. -/// -/// A delete is itself a write: LMDB copies the B-tree path before it frees the -/// leaf pages, and it may need a page for the free-list's own bookkeeping. On a -/// map pinned exactly to the file size a delete therefore has nowhere to go, -/// and the node could not prune its way back to health. -/// -/// Granted **only** on the delete retry path and taken away again inside the -/// same locked scope, so an ordinary store can never allocate from it. Leaving -/// it permanently in the ceiling would hand every node a little more of the -/// very reserve this mode exists to protect, multiplied by the nodes sharing -/// the volume. -const DELETE_COW_SLACK: u64 = 256 * 1024; - -/// Total permanent file growth deletes may cause per low-disk episode. -/// -/// What actually needs bounding is *growth*, not grants. Most slack-assisted -/// deletes reuse pages already inside `data.mdb` and grow it by nothing, and -/// those must stay free: a node has to be able to prune indefinitely, and page -/// reuse is not reliably available to the very next delete because LMDB cannot -/// hand back pages a still-recent transaction freed. Charging per grant instead -/// of per byte stops a node pruning after its first assisted delete. -/// -/// Only bytes the file actually gained are charged here. Reset when the store -/// leaves no-growth mode. A rounding error against [`DEFAULT_DISK_RESERVE`]. -const DELETE_COW_GROWTH_BUDGET: u64 = 1024 * 1024; - -/// Configuration for LMDB storage. -#[derive(Debug, Clone)] -pub struct LmdbStorageConfig { - /// Root directory for storage (LMDB env lives at `{root_dir}/chunks.mdb/`). - pub root_dir: PathBuf, - /// Whether to verify content on read (compares hash to address). - pub verify_on_read: bool, - /// Explicit LMDB map size cap in bytes. - /// - /// When 0 (default), the map size is computed automatically from available - /// disk space and grows on demand when more storage becomes available. - pub max_map_size: usize, - /// Minimum free disk space (in bytes) to preserve on the storage partition. - /// - /// Writes are refused when available space drops below this threshold. - pub disk_reserve: u64, -} - -impl Default for LmdbStorageConfig { - fn default() -> Self { - Self { - root_dir: PathBuf::from(".ant/chunks"), - verify_on_read: true, - max_map_size: 0, - disk_reserve: DEFAULT_DISK_RESERVE, - } - } -} - -impl LmdbStorageConfig { - /// A test-friendly default with `disk_reserve` set to 0 so unit tests - /// don't depend on the host having >= 1 GiB free disk space. - #[cfg(any(test, feature = "test-utils"))] - #[must_use] - pub fn test_default() -> Self { - Self { - disk_reserve: 0, - ..Self::default() - } - } -} - -/// Content-addressed LMDB storage. -/// -/// Uses heed (LMDB wrapper) for memory-mapped, transactional chunk storage. -/// Keys are 32-byte `XorName` addresses, values are raw chunk bytes. -pub struct LmdbStorage { - /// LMDB environment. - env: Env, - /// The unnamed default database (key=XorName bytes, value=chunk bytes). - db: Database, - /// Storage configuration. - config: LmdbStorageConfig, - /// Path to the LMDB environment directory (for disk-space queries). - env_dir: PathBuf, - /// Operation statistics. - stats: parking_lot::RwLock, - /// Serialises access to the LMDB environment during a map resize. - /// - /// Normal read/write operations acquire a **shared** lock. The rare - /// resize path acquires an **exclusive** lock, ensuring no transactions - /// are active when `env.resize()` is called (an LMDB safety requirement). - env_lock: Arc>, - /// Timestamp of the last successful disk-space check. - /// - /// `None` means "never checked — check on next write". Updated only - /// after a passing check, so a low-space result is always rechecked. - last_disk_ok: parking_lot::Mutex>, - /// Whether the map is currently pinned to the file's high-water mark. - /// - /// Set once available disk drops below the reserve. While pinned, LMDB can - /// still serve a write from its own free list but cannot extend - /// `data.mdb`, so the reserve is preserved by the allocator itself rather - /// than by refusing every write up front. - no_growth: Arc, - /// Keep the map pinned whatever the free space says. - /// - /// Set for the whole of the migration bridge. While both stores are open they each - /// measure the same free space and neither knows what the other is about to spend, so - /// a chunk written to both can be admitted twice against one lot of headroom and the - /// pair can cross the reserve together. Pinned, this environment cannot claim any new - /// disk at all: a write it cannot satisfy from its own free list is refused, and the - /// caller stores the chunk in files alone. That is the right answer anyway, because - /// this copy exists to make a rollback survivable, not to be the one that must - /// succeed. - growth_pinned: Arc, - /// Serialises entering and leaving no-growth mode. - /// - /// Setting `no_growth` and resizing the map is one compound transition - /// spanning an await. Without this, two callers straddling the threshold - /// can interleave so the flag ends up describing a map size that was never - /// applied, leaving the store unpinned while it believes it is pinned. - growth_mode_lock: tokio::sync::Mutex<()>, - /// Bytes `data.mdb` has permanently gained to slack-assisted deletes in - /// this low-disk episode. - /// - /// A delete's copy-on-write can extend the file, and LMDB never gives file - /// space back, so that growth is permanent. Bounding it stops repeated - /// fill-then-delete cycles walking the file into the reserve. Deletes that - /// find room inside the file cost nothing. Reset on leaving no-growth mode. - delete_growth_charged: Arc, - /// Tracks every LMDB blocking task spawned by this storage. - /// - /// A `spawn_blocking` closure owns a cloned [`Env`] and keeps running - /// even when its async awaiter is dropped (e.g. by a `select!` losing to - /// a shutdown token). Tracking the blocking task itself — not the async - /// wrapper — lets [`Self::wait_idle`] wait for true quiescence before - /// the environment may be reopened. - blocking_tracker: TaskTracker, - /// Test-only gate read-acquired at the top of the put blocking closure. - /// - /// Tests hold the write half to deterministically park an in-flight put - /// on the blocking pool (e.g. to prove [`Self::wait_idle`] waits for a - /// detached write). - #[cfg(any(test, feature = "test-utils"))] - test_put_gate: Arc>, - /// Test-only gate read-acquired inside the raw-read blocking closures, - /// immediately after the shared `env_lock` guard is taken. - /// - /// Tests hold the write half to deterministically park an in-flight raw - /// read while it still holds the shared environment lock (e.g. to prove - /// [`Self::try_resize`] waits for active raw reads before calling - /// `env.resize()`). - #[cfg(any(test, feature = "test-utils"))] - test_read_gate: Arc>, -} - -impl LmdbStorage { - /// Create a new LMDB storage instance. - /// - /// Opens (or creates) an LMDB environment at `{root_dir}/chunks.mdb/`. - /// - /// When `config.max_map_size` is 0 (the default) the map size is derived - /// from the available disk space on the partition that hosts the database, - /// minus `config.disk_reserve`. This allows a node to use all available - /// storage without a fixed cap. If the operator adds more storage later - /// the map is resized on demand (see [`Self::put`]). - /// - /// # Errors - /// - /// Returns an error if the LMDB environment cannot be opened. - #[allow(unsafe_code)] - pub async fn new(config: LmdbStorageConfig) -> Result { - let env_dir = config.root_dir.join("chunks.mdb"); - - // Create the directory synchronously before opening LMDB - std::fs::create_dir_all(&env_dir) - .map_err(|e| Error::Storage(format!("Failed to create LMDB directory: {e}")))?; - - let map_size = if config.max_map_size > 0 { - // Operator provided an explicit cap. - config.max_map_size - } else { - // Auto-scale: current DB footprint + available space − reserve. - let computed = compute_map_size(&env_dir, config.disk_reserve)?; - info!( - "Auto-computed LMDB map size: {:.2} GiB (data + available disk minus {:.2} GiB \ - reserve, head-room capped on Windows to bound page-table overhead)", - bytes_to_gib(computed as u64), - bytes_to_gib(config.disk_reserve), - ); - computed - }; - - let env_dir_clone = env_dir.clone(); - // Constructor-only blocking task: it runs before `self` (and its - // `blocking_tracker`) exists, so it is deliberately untracked. The - // constructor awaits it right here, so it cannot outlive this call. - let (env, db) = spawn_blocking(move || -> Result<(Env, Database)> { - // SAFETY: `EnvOpenOptions::open()` is unsafe because LMDB uses memory-mapped - // I/O and relies on OS file-locking to prevent corruption from concurrent - // access by multiple processes. We satisfy this by giving each node instance - // a unique `root_dir` (typically a directory named by its full 64-hex peer - // ID), ensuring no two processes open the same LMDB environment. Callers - // who manually configure `--root-dir` must not point multiple nodes at the - // same directory. - let env = unsafe { - EnvOpenOptions::new() - .map_size(map_size) - .max_dbs(1) - .open(&env_dir_clone) - .map_err(|e| Error::Storage(format!("Failed to open LMDB env: {e}")))? - }; - - let mut wtxn = env - .write_txn() - .map_err(|e| Error::Storage(format!("Failed to create write txn: {e}")))?; - let db: Database = env - .create_database(&mut wtxn, None) - .map_err(|e| Error::Storage(format!("Failed to create database: {e}")))?; - wtxn.commit() - .map_err(|e| Error::Storage(format!("Failed to commit db creation: {e}")))?; - - Ok((env, db)) - }) - .await - .map_err(|e| Error::Storage(format!("LMDB init task failed: {e}")))??; - - let storage = Self { - env, - db, - config, - env_dir, - stats: parking_lot::RwLock::new(StorageStats::default()), - env_lock: Arc::new(parking_lot::RwLock::new(())), - last_disk_ok: parking_lot::Mutex::new(None), - no_growth: Arc::new(AtomicBool::new(false)), - growth_pinned: Arc::new(AtomicBool::new(false)), - growth_mode_lock: tokio::sync::Mutex::new(()), - delete_growth_charged: Arc::new(AtomicU64::new(0)), - blocking_tracker: TaskTracker::new(), - #[cfg(any(test, feature = "test-utils"))] - test_put_gate: Arc::new(parking_lot::RwLock::new(())), - #[cfg(any(test, feature = "test-utils"))] - test_read_gate: Arc::new(parking_lot::RwLock::new(())), - }; - - debug!( - "Initialized LMDB storage at {:?} ({} existing chunks)", - storage.env_dir, - storage.current_chunks()? - ); - - Ok(storage) - } - - /// Store a chunk. - /// - /// Before writing, verifies that available disk space exceeds the - /// configured reserve. If the LMDB map is full but more disk space - /// exists (e.g. the operator added storage), the map is resized - /// automatically and the write is retried. - /// - /// # Returns - /// - /// Returns `true` if the chunk was newly stored, `false` if it already existed. - /// - /// # Errors - /// - /// Returns an error if the write fails, content doesn't match address, - /// or the disk is too full to accept new chunks. - pub async fn put(&self, address: &XorName, content: &[u8]) -> Result { - self.put_inner(address, content, true).await - } - - /// Store bytes under a key they do not hash to. Tests only. - /// - /// Stands in for a record that rotted in place, which is the one shape the ordinary - /// path refuses to create and the migration has to survive finding. - /// - /// # Errors - /// - /// As [`Self::put`], minus the address check. - #[cfg(test)] - pub(crate) async fn put_unchecked(&self, address: &XorName, content: &[u8]) -> Result { - self.put_inner(address, content, false).await - } - - async fn put_inner(&self, address: &XorName, content: &[u8], verify: bool) -> Result { - // Verify content address - let computed = Self::compute_address(content); - if verify && computed != *address { - return Err(Error::Storage(format!( - "Content address mismatch: expected {}, computed {}", - hex::encode(address), - hex::encode(computed) - ))); - } - - // Fast-path duplicate check (read-only, no write lock needed). - // This is an optimistic hint — the authoritative check happens inside - // the write transaction below to prevent TOCTOU races. - if self.exists(address)? { - trace!("Chunk {} already exists", hex::encode(address)); - self.stats.write().duplicates += 1; - return Ok(false); - } - - // ── Capacity guard (cached — at most one syscall per interval) ── - // Placed after the duplicate check so that re-storing an existing - // chunk remains a harmless no-op even when disk space is low. - // - // Below the reserve this pins the map instead of refusing outright, so - // the write is still attempted and LMDB decides whether a freed page - // can take it. A node that has pruned heavily keeps serving the network - // from the space it already occupies. - let no_growth = self.sync_growth_mode().await?; - - // ── Write (with resize-on-demand) ─────────────────────────────── - match self.try_put(address, content).await? { - PutOutcome::New => {} - PutOutcome::Duplicate => { - trace!("Chunk {} already exists", hex::encode(address)); - self.stats.write().duplicates += 1; - return Ok(false); - } - PutOutcome::MapFull if no_growth => { - // Both halves are now true: the volume is below the reserve and - // no free page can take *this* value. Resizing would extend the - // file into the reserve, so refuse. - // - // The refusal is not remembered. `MapFull` is specific to the - // size just attempted — a smaller value may still fit a smaller - // run — so caching it would let one maximum-sized chunk lock out - // every subsequent write. `check_capacity` estimates instead. - return Err(Error::Storage(format!( - "Insufficient disk space: {:.2} GiB reserve required and no reusable page \ - in the local store fits this {} B value. \ - Free disk space or increase the partition to continue storing chunks.", - bytes_to_gib(self.config.disk_reserve), - content.len(), - ))); - } - PutOutcome::MapFull => { - // The map ceiling was reached but there may be more disk space - // available (e.g. operator expanded the partition). - // - // Guarded: `no_growth` was sampled before the write, so the - // store may have entered no-growth mode since. Growing the map - // outside the transition lock could undo a pin that a - // concurrent `sync_growth_mode` had just applied, handing the - // reserve back to ordinary writes. - self.try_resize_for_growth().await?; - // Retry once after resize. - match self.try_put(address, content).await? { - PutOutcome::New => {} - PutOutcome::Duplicate => { - self.stats.write().duplicates += 1; - return Ok(false); - } - PutOutcome::MapFull => { - return Err(Error::Storage( - "LMDB map full after resize — disk may be at capacity".into(), - )); - } - } - } - } - - { - let mut stats = self.stats.write(); - stats.chunks_stored += 1; - stats.bytes_stored += content.len() as u64; - } - - debug!( - "Stored chunk {} ({} bytes)", - hex::encode(address), - content.len() - ); - - Ok(true) - } - - /// Attempt a single put inside a write transaction. - /// - /// Returns [`PutOutcome::MapFull`] instead of an error when the LMDB map - /// ceiling is reached, so the caller can resize and retry. - async fn try_put(&self, address: &XorName, content: &[u8]) -> Result { - let key = *address; - let value = content.to_vec(); - let env = self.env.clone(); - let db = self.db; - let lock = Arc::clone(&self.env_lock); - #[cfg(any(test, feature = "test-utils"))] - let test_put_gate = Arc::clone(&self.test_put_gate); - - self.blocking_tracker - .spawn_blocking(move || -> Result { - // Test-only: parks here while a test holds the write half. - #[cfg(any(test, feature = "test-utils"))] - let _test_put_gate = test_put_gate.read(); - let _guard = lock.read(); - - let mut wtxn = env - .write_txn() - .map_err(|e| Error::Storage(format!("Failed to create write txn: {e}")))?; - - // Authoritative existence check inside the serialized write txn - if db - .get(&wtxn, &key) - .map_err(|e| Error::Storage(format!("Failed to check existence: {e}")))? - .is_some() - { - return Ok(PutOutcome::Duplicate); - } - - match db.put(&mut wtxn, &key, &value) { - Ok(()) => {} - Err(heed::Error::Mdb(MdbError::MapFull)) => return Ok(PutOutcome::MapFull), - Err(e) => { - return Err(Error::Storage(format!("Failed to put chunk: {e}"))); - } - } - - match wtxn.commit() { - Ok(()) => Ok(PutOutcome::New), - Err(heed::Error::Mdb(MdbError::MapFull)) => Ok(PutOutcome::MapFull), - Err(e) => Err(Error::Storage(format!("Failed to commit put: {e}"))), - } - }) - .await - .map_err(|e| Error::Storage(format!("LMDB put task failed: {e}")))? - } - - /// Retrieve a chunk. - /// - /// # Returns - /// - /// Returns `Some(content)` if found, `None` if not found. - /// - /// # Errors - /// - /// Returns an error if read fails or verification fails. - pub async fn get(&self, address: &XorName) -> Result>> { - let key = *address; - let env = self.env.clone(); - let db = self.db; - let lock = Arc::clone(&self.env_lock); - - let content = self - .blocking_tracker - .spawn_blocking(move || -> Result>> { - let _guard = lock.read(); - let rtxn = env - .read_txn() - .map_err(|e| Error::Storage(format!("Failed to create read txn: {e}")))?; - let value = db - .get(&rtxn, &key) - .map_err(|e| Error::Storage(format!("Failed to get chunk: {e}")))?; - Ok(value.map(Vec::from)) - }) - .await - .map_err(|e| Error::Storage(format!("LMDB get task failed: {e}")))??; - - let Some(content) = content else { - trace!("Chunk {} not found", hex::encode(address)); - return Ok(None); - }; - - // Verify content if configured - if self.config.verify_on_read { - let computed = Self::compute_address(&content); - if computed != *address { - self.stats.write().verification_failures += 1; - warn!( - "Chunk verification failed: expected {}, computed {}", - hex::encode(address), - hex::encode(computed) - ); - return Err(Error::Storage(format!( - "Chunk verification failed for {}", - hex::encode(address) - ))); - } - } - - { - let mut stats = self.stats.write(); - stats.chunks_retrieved += 1; - stats.bytes_retrieved += content.len() as u64; - } - - debug!( - "Retrieved chunk {} ({} bytes)", - hex::encode(address), - content.len() - ); - - Ok(Some(content)) - } - - /// Check if a chunk exists. - /// - /// # Errors - /// - /// Returns an error if the LMDB read transaction fails. - pub fn exists(&self, address: &XorName) -> Result { - let _guard = self.env_lock.read(); - let rtxn = self - .env - .read_txn() - .map_err(|e| Error::Storage(format!("Failed to create read txn: {e}")))?; - let found = self - .db - .get(&rtxn, address.as_ref()) - .map_err(|e| Error::Storage(format!("Failed to check existence: {e}")))? - .is_some(); - Ok(found) - } - - /// Delete a chunk. - /// - /// # Errors - /// - /// Returns an error if deletion fails. - pub async fn delete(&self, address: &XorName) -> Result { - let key = *address; - - // Establish growth mode first, exactly as `put` does. Otherwise a - // delete arriving while the volume is low but before any write has - // pinned the map would copy-on-write into whatever head-room the - // ceiling still had, growing `data.mdb` into the reserve without - // passing through the budgeted allowance below. - self.sync_growth_mode().await?; - - let deleted = match self.try_delete(&key).await? { - DeleteOutcome::Done(existed) => existed, - DeleteOutcome::MapFull => { - // A delete is a write: LMDB copies the B-tree path before it - // frees the leaf pages, so a store with no free page at all - // cannot delete inside a map pinned to the file size. Without a - // way through, a node that filled up before it ever pruned - // could never prune its way out. - // - // Serialised against `sync_growth_mode` so the two cannot - // interleave their resizes. - let _transition = self.growth_mode_lock.lock().await; - self.delete_with_slack(&key).await? - } - }; - - if deleted { - debug!("Deleted chunk {}", hex::encode(address)); - } - - Ok(deleted) - } - - /// Attempt one delete, reporting `MapFull` rather than raising it. - async fn try_delete(&self, key: &XorName) -> Result { - let key = *key; - let env = self.env.clone(); - let db = self.db; - let lock = Arc::clone(&self.env_lock); - - self.blocking_tracker - .spawn_blocking(move || -> Result { - let _guard = lock.read(); - delete_in_txn(&env, db, &key) - }) - .await - .map_err(|e| Error::Storage(format!("LMDB delete task failed: {e}")))? - } - - /// Get storage statistics. - #[must_use] - pub fn stats(&self) -> StorageStats { - let mut stats = self.stats.read().clone(); - match self.current_chunks() { - Ok(count) => stats.current_chunks = count, - Err(e) => { - warn!("Failed to read current_chunks for stats: {e}"); - stats.current_chunks = 0; - } - } - stats - } - - /// Return the number of chunks currently stored, queried from LMDB metadata. - /// - /// This is an O(1) read of the B-tree page header — not a full scan. - /// - /// # Errors - /// - /// Returns an error if the LMDB read transaction fails. - pub fn current_chunks(&self) -> Result { - let _guard = self.env_lock.read(); - let rtxn = self - .env - .read_txn() - .map_err(|e| Error::Storage(format!("Failed to create read txn: {e}")))?; - let entries = self - .db - .stat(&rtxn) - .map_err(|e| Error::Storage(format!("Failed to read db stats: {e}")))? - .entries; - Ok(entries as u64) - } - - /// Compute content address (BLAKE3 hash). - #[must_use] - pub fn compute_address(content: &[u8]) -> XorName { - crate::client::compute_address(content) - } - - /// Get the root directory. - #[must_use] - pub fn root_dir(&self) -> &Path { - &self.config.root_dir - } - - /// Return all stored record keys. - /// - /// Iterates the LMDB database in a read transaction. Used by the - /// replication subsystem for hint construction and audit sampling. - /// - /// # Errors - /// - /// Returns an error if the LMDB read transaction fails. - pub async fn all_keys(&self) -> Result> { - let env = self.env.clone(); - let db = self.db; - let lock = Arc::clone(&self.env_lock); - - let keys = self - .blocking_tracker - .spawn_blocking(move || -> Result> { - // Hold the shared lock for the whole read so try_resize() (which - // takes the exclusive lock before the unsafe Env::resize()) cannot - // unmap the environment while this txn and its cursor are live. - let _guard = lock.read(); - let rtxn = env - .read_txn() - .map_err(|e| Error::Storage(format!("Failed to create read txn: {e}")))?; - let mut keys = Vec::new(); - let iter = db - .iter(&rtxn) - .map_err(|e| Error::Storage(format!("Failed to iterate database: {e}")))?; - for result in iter { - let (key_bytes, _) = - result.map_err(|e| Error::Storage(format!("Failed to read entry: {e}")))?; - if key_bytes.len() == XORNAME_LEN { - let mut key = [0u8; XORNAME_LEN]; - key.copy_from_slice(key_bytes); - keys.push(key); - } else { - crate::logging::warn!( - "LmdbStorage: skipping entry with unexpected key length {} (expected {XORNAME_LEN})", - key_bytes.len() - ); - } - } - Ok(keys) - }) - .await - .map_err(|e| Error::Storage(format!("all_keys task failed: {e}")))?; - - keys - } - - /// Retrieve raw chunk bytes without content-address verification. - /// - /// Used by the audit subsystem to compute digests over stored bytes. - /// Unlike [`Self::get`], this does not verify `hash(content) == address`. - /// - /// # Errors - /// - /// Returns an error if the LMDB read transaction fails. - pub async fn get_raw(&self, address: &XorName) -> Result>> { - let key = *address; - let env = self.env.clone(); - let db = self.db; - let lock = Arc::clone(&self.env_lock); - #[cfg(any(test, feature = "test-utils"))] - let test_read_gate = Arc::clone(&self.test_read_gate); - - let value = self - .blocking_tracker - .spawn_blocking(move || -> Result>> { - // Shared lock held until the bytes are copied out, so a concurrent - // try_resize() cannot unmap the environment mid-read. See all_keys. - let _guard = lock.read(); - // Test-only: parks here, still holding the shared lock, while a - // test holds the write half — used to prove a resize waits. - #[cfg(any(test, feature = "test-utils"))] - let _test_read_gate = test_read_gate.read(); - let rtxn = env - .read_txn() - .map_err(|e| Error::Storage(format!("Failed to create read txn: {e}")))?; - let val = db - .get(&rtxn, key.as_ref()) - .map_err(|e| Error::Storage(format!("Failed to get chunk: {e}")))?; - Ok(val.map(Vec::from)) - }) - .await - .map_err(|e| Error::Storage(format!("get_raw task failed: {e}")))?; - - value - } - - /// Cheap capacity pre-check for callers that want to reject work *before* - /// doing expensive setup (e.g. the PUT handler skipping payment - /// verification on a full node — see `V2-411`). - /// - /// A node is full only when **both** halves are true: the volume is below - /// the reserve *and* the store has no reusable page left. Deleting a record - /// returns its pages to LMDB's free list and never to the filesystem, so a - /// node that has pruned heavily sits on reusable capacity while `statvfs` - /// still reports the volume as full. Refusing on the disk half alone stops - /// such a node from writing into space it already owns. - /// - /// This is a **hint**, deliberately biased towards admitting: it estimates - /// reusable bytes and only refuses when there is not even one chunk's worth. - /// The authority on whether a given write fits stays with LMDB's allocator - /// in [`Self::put`], because no page count can account for the - /// copy-on-write of the B-tree path, the contiguous run a multi-megabyte - /// value needs, or pages still pinned by an open read transaction. An - /// over-optimistic hint costs one refused write; an over-pessimistic one - /// would recreate the bug this exists to fix. - /// - /// # Errors - /// - /// Returns [`Error::Storage`] when the volume is below the reserve and the - /// store holds less than one chunk of reusable space, or when the - /// disk-space query itself fails. - /// Unused while the node is moving off this store. - /// - /// Capacity is now the file store's question, because that is where writes land, and - /// this predicate deliberately answers a different one: it counts pages this store can - /// reuse internally, which says nothing about whether the *file* about to be written - /// will fit. [`Self::capacity_verdict`] is still used, to decide whether the bridge's - /// copy into this store is worth attempting. Both go when this store does. - #[allow(dead_code)] - pub(crate) fn check_capacity(&self) -> Result<()> { - let Some(available) = self.available_space_cached()? else { - return Ok(()); - }; - - let reusable = self.reusable_bytes()?; - if reusable >= MAX_CHUNK_SIZE as u64 { - return Ok(()); - } - - Err(Error::Storage(format!( - "Insufficient disk space: {:.2} GiB available, {:.2} GiB reserve required, \ - and only {reusable} B reusable inside the local store. \ - Free disk space or increase the partition to continue storing chunks.", - bytes_to_gib(available), - bytes_to_gib(self.config.disk_reserve), - ))) - } - - /// Capacity as a three-way verdict, distinguishing a full node from a - /// query that failed. - /// - /// Refuses on the same two-part predicate as [`Self::check_capacity`]: - /// `Full` only when the volume is below the reserve *and* the store holds - /// less than one chunk of reusable space. Deleted records return their - /// pages to LMDB's free list and never to the filesystem, so a pruned node - /// reads as full to `statvfs` while still able to store chunks — such a - /// node must keep discovering holders for the keys it owes. - /// - /// Shares the same TTL cache as [`Self::check_capacity`]: a passing disk - /// reading is cached, a failing one is always rechecked, so freed space is - /// noticed promptly. An admit that rests on reusable pages is deliberately - /// not cached, matching the pre-check, because the free list can drain a - /// chunk at a time. - pub(crate) fn capacity_verdict(&self) -> CapacityVerdict { - { - let last = self.last_disk_ok.lock(); - if let Some(t) = *last { - if t.elapsed().as_secs() < DISK_CHECK_INTERVAL_SECS { - return CapacityVerdict::Writable; - } - } - } - let disk = verdict_from_available_space( - fs2::available_space(&self.env_dir), - self.config.disk_reserve, - ); - match disk { - CapacityVerdict::Writable => { - *self.last_disk_ok.lock() = Some(Instant::now()); - CapacityVerdict::Writable - } - CapacityVerdict::Unknown => CapacityVerdict::Unknown, - // Below the reserve is only half the predicate: the store may - // still hold pages it can reuse without growing the file. - CapacityVerdict::Full => match self.reusable_bytes() { - Ok(reusable) if reusable >= MAX_CHUNK_SIZE as u64 => CapacityVerdict::Writable, - Ok(_) => CapacityVerdict::Full, - Err(e) => { - warn!("Could not query the store's reusable space: {e}"); - CapacityVerdict::Unknown - } - }, - } - } - - /// Estimated bytes inside `data.mdb` that LMDB could write without growing - /// the file: the file size minus the pages currently holding data. - /// - /// Deliberately an over-estimate. `stat()` counts only the branch, leaf and - /// overflow pages of the unnamed database, so the free-list's own pages and - /// the environment metadata fall on the "reusable" side. Erring high keeps - /// [`Self::check_capacity`] biased towards admitting the attempt. - /// - /// Uses `stat()` rather than heed's `non_free_pages_size()`, which walks the - /// unnamed database calling `String::from_utf8(key).unwrap()` on every key - /// without a zero byte. Our keys are 32 random bytes, so that call panics - /// almost immediately. A single unnamed database makes `stat()` equivalent. - fn reusable_bytes(&self) -> Result { - // Order matters. The two samples are not atomic, so read the live pages - // first and the file length second: a write committing in between then - // pairs an older (smaller) live count with a newer (larger) file, which - // over-estimates. Sampling the other way round pairs a stale file - // length with a fresh live count and can under-estimate, which would - // refuse a node that has room — the very bug this fixes. - let stat = self.env.stat(); - let live_pages = (stat.branch_pages as u64) - .saturating_add(stat.leaf_pages as u64) - .saturating_add(stat.overflow_pages as u64); - let live_bytes = live_pages.saturating_mul(u64::from(stat.page_size)); - - let file_bytes = self - .env - .real_disk_size() - .map_err(|e| Error::Storage(format!("Failed to query LMDB file size: {e}")))?; - - Ok(file_bytes.saturating_sub(live_bytes)) - } - - /// Available bytes on the storage volume, or `None` when a recent check - /// already showed it above the reserve. - /// - /// Only *passing* results are cached, so a low-space condition is always - /// re-measured and freed space is detected promptly. - fn available_space_cached(&self) -> Result> { - { - let last = self.last_disk_ok.lock(); - if let Some(t) = *last { - if t.elapsed().as_secs() < DISK_CHECK_INTERVAL_SECS { - return Ok(None); - } - } - } - - let available = fs2::available_space(&self.env_dir) - .map_err(|e| Error::Storage(format!("Failed to query available disk space: {e}")))?; - - if available >= self.config.disk_reserve { - *self.last_disk_ok.lock() = Some(Instant::now()); - return Ok(None); - } - - Ok(Some(available)) - } - - /// Align the map ceiling with the current disk state, returning whether the - /// store is in no-growth mode. - /// - /// Below the reserve the map is pinned to the file's high-water mark, so a - /// put succeeds exactly when LMDB can satisfy it from the free list and - /// returns `MapFull` the moment it would need to extend `data.mdb`. That - /// makes the allocator the authority on "can this write fit". - /// - /// The whole transition runs under `growth_mode_lock`. Setting the flag and - /// resizing the map is one compound change spanning an await, so without - /// serialisation two callers straddling the threshold can interleave and - /// leave the flag describing a map that was never applied. - async fn sync_growth_mode(&self) -> Result { - let _transition = self.growth_mode_lock.lock().await; - - // Re-measured inside the lock: a caller that queued behind a transition - // must act on the state that transition left behind, not the one it saw - // before waiting. - // Pinned for the bridge: never unpinned by having room, because the room is not - // this environment's to spend while another store is measuring the same disk. - if self.growth_pinned.load(Ordering::Acquire) { - self.no_growth.store(true, Ordering::Release); - self.pin_map_to_high_water().await?; - return Ok(true); - } - if self.available_space_cached()?.is_none() { - // At or above the reserve: restore normal head-room if we pinned it. - if self.no_growth.load(Ordering::Acquire) { - // Intent first, work second. A `spawn_blocking` body outlives a - // cancelled awaiter, so ordering between two resizes cannot be - // guaranteed by holding an async lock. Publishing the intent - // before the work lets each closure re-read it under the - // exclusive lock and decline if it has since been reversed. - self.no_growth.store(false, Ordering::Release); - self.try_resize().await?; - } - // Real disk again: the maintenance allowance is refreshed. Done on - // every healthy pass, not just the transition, so an allowance - // spent while the flag happened to be clear is still returned. - self.delete_growth_charged.store(0, Ordering::Release); - return Ok(false); - } - - // Called unconditionally, not just on the transition. A re-pin that - // failed, or a transition whose caller was cancelled while its detached - // resize was still in flight, can leave the flag set while the map is - // not actually pinned; re-asserting it here repairs that instead of - // trusting the flag. The call is a no-op when already pinned. - self.no_growth.store(true, Ordering::Release); - self.pin_map_to_high_water().await?; - - Ok(true) - } - - /// Keep this environment from ever claiming new disk, until the process ends. - /// - /// For the migration bridge, where a second store measures the same free space and - /// neither knows what the other is about to spend. Pinned, this one writes only from - /// pages it already holds, so the other's accounting is the only claim on free disk. - /// - /// # Errors - /// - /// Returns [`Error::Storage`] if the map cannot be pinned. - pub async fn pin_growth(&self) -> Result<()> { - self.growth_pinned.store(true, Ordering::Release); - self.sync_growth_mode().await.map(|_| ()) - } - - /// Pin the LMDB map to the size of `data.mdb` on disk. - /// - /// Every page already in the file stays usable, including free ones, but - /// the file cannot grow, so the configured reserve is preserved by LMDB - /// itself rather than by refusing writes it could have served. - /// - /// Deliberately leaves **no** head-room: any slack in the ceiling is - /// ordinary put capacity, so it would be spent on the next chunk rather - /// than kept for maintenance, and on a shared volume every node would take - /// its own slice out of the reserve. Deletes get their copy-on-write room - /// on demand instead, see [`Self::delete`]. - /// - /// Takes the **exclusive** `env_lock` for the same reason - /// [`Self::try_resize`] does: `mdb_env_set_mapsize` requires that no - /// transaction is active. Callers hold `growth_mode_lock`. - #[allow(unsafe_code)] - async fn pin_map_to_high_water(&self) -> Result<()> { - // The "is it already pinned?" test lives inside the exclusive lock - // below, not out here. An unlocked pre-check can observe "already - // pinned" moments before a detached resize from a cancelled transition - // lands, after which the flag would claim a pin that no longer holds. - // Callers invoke this on every low-disk write so the pinned state - // repairs itself; the locked section is a few reads when nothing is to - // be done. - let env = self.env.clone(); - let lock = Arc::clone(&self.env_lock); - let no_growth = Arc::clone(&self.no_growth); - - self.blocking_tracker - .spawn_blocking(move || -> Result<()> { - // Exclusive lock guarantees no concurrent transactions. - let _guard = lock.write(); - - // Re-read under the lock: this closure may have been queued - // behind others, or its awaiter cancelled, and the store may - // have left no-growth mode since it was spawned. - if !no_growth.load(Ordering::Acquire) { - return Ok(()); - } - - let current_map = env.info().map_size; - let file_bytes = env - .real_disk_size() - .map_err(|e| Error::Storage(format!("Failed to query LMDB file size: {e}")))?; - - let page = page_size::get() as u64; - let aligned = file_bytes.div_ceil(page) * page; - let target = usize::try_from(aligned).unwrap_or(usize::MAX); - - // Re-checked under the lock: the state may have moved between - // the cheap check and here. - if target >= current_map { - return Ok(()); - } - - // SAFETY: We hold an exclusive lock, so no transactions are active. - unsafe { - env.resize(target) - .map_err(|e| Error::Storage(format!("Failed to pin LMDB map: {e}")))?; - } - - info!( - "Disk below reserve: pinned LMDB map to {:.2} GiB (was {:.2} GiB). \ - Writes that fit in already-freed pages still succeed; \ - only writes that would grow the file are refused.", - bytes_to_gib(target as u64), - bytes_to_gib(current_map as u64), - ); - Ok(()) - }) - .await - .map_err(|e| Error::Storage(format!("LMDB map pin task failed: {e}")))? - } - - /// Grow the map for a write, unless the store is pinned below the reserve. - /// - /// Serialised against [`Self::sync_growth_mode`] so a resize cannot land - /// after a pin and quietly undo it. If the store entered no-growth mode - /// while the write was in flight, the caller's `MapFull` is final and no - /// growth happens. - async fn try_resize_for_growth(&self) -> Result<()> { - let _transition = self.growth_mode_lock.lock().await; - - if self.no_growth.load(Ordering::Acquire) { - return Ok(()); - } - - self.try_resize().await - } - - /// Delete `key` with [`DELETE_COW_SLACK`] of temporary map head-room, then - /// take the head-room straight back. - /// - /// The raise, the delete and the re-pin all happen inside **one** exclusive - /// `env_lock` scope. Doing them as three separate locked steps would leave - /// windows in which an ordinary put could allocate from the raised ceiling, - /// spending the reserve on a chunk instead of on the maintenance it was - /// granted for, and an error or cancellation between the steps would leave - /// the ceiling raised for good. - /// - /// What is budgeted is the *growth*, not the grant. If the copy-on-write - /// does extend `data.mdb` that growth is permanent, since LMDB never - /// returns file space, so repeated fill-then-delete cycles could otherwise - /// walk the file into the reserve a slice at a time. A delete that finds - /// room inside the file is charged nothing. - /// - /// Charging per grant instead would be wrong, and was: page reuse is not - /// reliably available to the very next delete, because LMDB will not hand - /// back pages a still-recent transaction freed. A one-grant budget - /// therefore stopped a node pruning after its first assisted delete, which - /// showed up as every delete failing on 4 KiB-page hosts while passing on - /// 16 KiB-page ones. The budget resets when the store leaves no-growth - /// mode, i.e. when there is real disk to work with again. - #[allow(unsafe_code)] - async fn delete_with_slack(&self, key: &XorName) -> Result { - let key = *key; - let env = self.env.clone(); - let db = self.db; - let lock = Arc::clone(&self.env_lock); - let budget = Arc::clone(&self.delete_growth_charged); - - let outcome = self - .blocking_tracker - .spawn_blocking(move || -> Result { - // Checked and charged entirely inside the closure. A - // `spawn_blocking` body keeps running when its awaiter is - // dropped, so accounting split across the await could be - // skipped, permanently costing the node its ability to prune. - if budget.load(Ordering::Acquire) >= DELETE_COW_GROWTH_BUDGET { - return Err(Error::Storage(format!( - "Cannot delete: the local store is full and deletes have already used \ - their {DELETE_COW_GROWTH_BUDGET} B growth allowance. \ - Free disk space to continue." - ))); - } - - // Exclusive for the whole sequence: no transaction may be - // active across either resize, and no put may observe the - // raised ceiling. - let _guard = lock.write(); - - let page = page_size::get() as u64; - let previous_map = env.info().map_size; - let file_before = env - .real_disk_size() - .map_err(|e| Error::Storage(format!("Failed to query LMDB file size: {e}")))?; - let raised = (previous_map as u64) - .saturating_add(DELETE_COW_SLACK) - .div_ceil(page) - .saturating_mul(page); - - // SAFETY: exclusive lock held, so no transactions are active. - let granted = unsafe { - env.resize(usize::try_from(raised).unwrap_or(usize::MAX)) - .map_err(|e| Error::Storage(format!("Failed to grant delete slack: {e}"))) - }; - granted?; - - // Armed across the delete so an unwind still restores the - // ceiling; disarmed once the explicit restore below succeeds. - let mut ceiling_guard = MapCeilingRestorer { - env: &env, - previous: previous_map, - armed: true, - }; - - let outcome = delete_in_txn(&env, db, &key); - - // Charge what the file actually gained, not the fact that slack - // was offered. A delete that found room inside `data.mdb` costs - // nothing and must not consume the allowance, otherwise a node - // stops being able to prune after its first assisted delete. - // Measured before the ceiling is restored, and before any error - // is propagated, so a committed delete is always accounted for. - // - // A measurement that fails is charged the whole slack rather than nothing. - // Reading it back as the size before the delete would say the file did not - // grow, and a delete that did grow would then spend disk the budget never - // saw. Repeat that and the ceiling stops meaning anything. Over-charging - // costs at worst one assisted delete; under-charging costs the reserve. - let grew = match env.real_disk_size() { - Ok(file_after) => file_after.saturating_sub(file_before), - Err(e) => { - warn!( - "Could not measure the LMDB file after an assisted delete \ - ({e}); charging the whole slack rather than assuming it cost \ - nothing" - ); - DELETE_COW_SLACK - } - }; - if grew > 0 { - budget.fetch_add(grew, Ordering::AcqRel); - } - - // Undo the raise before releasing the lock, on every path and - // whatever the delete did. Restoring to the previous ceiling - // rather than to a freshly measured file size keeps this - // unconditional: it is exactly the inverse of the raise, needs - // no second syscall that could itself fail, and is correct - // whether or not the store was pinned. If the copy-on-write did - // extend the file, LMDB clamps a request below the space in use, - // so the map still covers the data. - // - // SAFETY: exclusive lock held, so no transactions are active. - let restored = unsafe { - env.resize(previous_map) - .map_err(|e| Error::Storage(format!("Failed to restore LMDB map: {e}"))) - }; - if restored.is_ok() { - ceiling_guard.armed = false; - } - - // A failed restore is reported ahead of a failed delete, so the - // failure is not lost behind the delete's own error. - match (outcome, restored) { - (Ok(outcome), Ok(())) => Ok(outcome), - (_, Err(e)) | (Err(e), Ok(())) => Err(e), - } - }) - .await - .map_err(|e| Error::Storage(format!("LMDB delete-slack task failed: {e}")))?; - - match outcome? { - DeleteOutcome::Done(existed) => Ok(existed), - DeleteOutcome::MapFull => Err(Error::Storage( - "LMDB map full during delete even with the maintenance allowance".into(), - )), - } - } - - /// Grow the LMDB map to match currently available disk space. - /// - /// The new size is the **larger** of: - /// 1. the current map size (so existing data is never truncated), and - /// 2. `current_db_file_size + available_space − reserve` - /// (so all reachable disk space can be used). - /// - /// Acquires an **exclusive** lock on `env_lock` so that no read or write - /// transactions are active when the underlying `mdb_env_set_mapsize` is - /// called (an LMDB safety requirement). - #[allow(unsafe_code)] - async fn try_resize(&self) -> Result<()> { - let env = self.env.clone(); - let lock = Arc::clone(&self.env_lock); - let no_growth = Arc::clone(&self.no_growth); - let env_dir = self.env_dir.clone(); - let reserve = self.config.disk_reserve; - - self.blocking_tracker - .spawn_blocking(move || -> Result<()> { - // Exclusive lock guarantees no concurrent transactions. - let _guard = lock.write(); - - // Re-read under the lock. A `spawn_blocking` body outlives a - // cancelled awaiter, so this closure may land after the store - // entered no-growth mode. Growing then would hand back the - // head-room a pin had just taken away, and with it the disk - // reserve. - if no_growth.load(Ordering::Acquire) { - return Ok(()); - } - - // Measured here rather than before the spawn, so a late closure - // sizes from the disk as it is now, not as it was when queued. - let from_disk = compute_map_size(&env_dir, reserve)?; - - // Never shrink below the current map — existing data must remain - // addressable regardless of what the disk-space calculation says. - let current_map = env.info().map_size; - let new_size = from_disk.max(current_map); - - if new_size <= current_map { - debug!("LMDB map resize skipped — no additional disk space available"); - return Ok(()); - } - - // SAFETY: We hold an exclusive lock, so no transactions are active. - unsafe { - env.resize(new_size) - .map_err(|e| Error::Storage(format!("Failed to resize LMDB map: {e}")))?; - } - - info!( - "Resized LMDB map to {:.2} GiB (was {:.2} GiB)", - bytes_to_gib(new_size as u64), - bytes_to_gib(current_map as u64), - ); - Ok(()) - }) - .await - .map_err(|e| Error::Storage(format!("LMDB resize task failed: {e}")))? - } - - /// Wait until every tracked LMDB blocking task has finished. - /// - /// Dropping an async caller (e.g. a `select!` losing to a shutdown token) - /// does not cancel an already-spawned blocking closure — the closure keeps - /// running on the blocking pool with a cloned [`Env`]. This method waits - /// for those detached closures too, so when it returns no blocking - /// operation still holds the environment. - /// - /// Quiescence is only meaningful once callers have stopped issuing new - /// operations; concurrent traffic can keep the tracker non-empty - /// indefinitely. The storage remains fully usable afterwards (the - /// internal tracker is reopened before returning). - pub async fn wait_idle(&self) { - self.blocking_tracker.close(); - self.blocking_tracker.wait().await; - self.blocking_tracker.reopen(); - } - - /// Test-only handle to the put gate. - /// - /// Hold the write half to deterministically park the next put inside its - /// blocking closure (e.g. to exercise [`Self::wait_idle`] with a write - /// still in flight). - #[cfg(any(test, feature = "test-utils"))] - #[must_use] - pub fn test_put_gate(&self) -> Arc> { - Arc::clone(&self.test_put_gate) - } - - /// Test-only handle to the raw-read gate. - /// - /// Hold the write half to deterministically park the next raw read - /// (`get_raw`) inside its blocking closure while it still holds the shared - /// environment lock (e.g. to prove `try_resize` waits for active - /// raw reads). - #[cfg(any(test, feature = "test-utils"))] - #[must_use] - pub fn test_read_gate(&self) -> Arc> { - Arc::clone(&self.test_read_gate) - } -} - -// ──────────────────────────────────────────────────────────────────────────── -// Helpers -// ──────────────────────────────────────────────────────────────────────────── - -/// Outcome of a single `try_put` attempt. -enum PutOutcome { - /// Chunk was newly stored. - New, - /// Chunk already existed (idempotent). - Duplicate, - /// The LMDB map ceiling was reached — caller should resize and retry. - MapFull, -} - -/// Restores an LMDB map ceiling when dropped, including while unwinding. -/// -/// The explicit restore in [`LmdbStorage::delete_with_slack`] is the normal -/// path, because it can report a failure to the caller. This exists so a panic -/// between the raise and that restore cannot leave the ceiling raised, which -/// would quietly hand ordinary writes the disk reserve. -struct MapCeilingRestorer<'a> { - env: &'a Env, - previous: usize, - armed: bool, -} - -impl Drop for MapCeilingRestorer<'_> { - #[allow(unsafe_code)] - fn drop(&mut self) { - if !self.armed { - return; - } - // SAFETY: the owner holds the exclusive `env_lock` for this whole - // scope, so no transaction is active. - unsafe { - if let Err(e) = self.env.resize(self.previous) { - warn!("Failed to restore the LMDB map ceiling while unwinding: {e}"); - } - } - } -} - -/// Run one delete in its own write transaction, reporting `MapFull` rather than -/// raising it. -/// -/// The caller owns the `env_lock` discipline: [`LmdbStorage::try_delete`] holds -/// the shared guard, [`LmdbStorage::delete_with_slack`] the exclusive one. -fn delete_in_txn(env: &Env, db: Database, key: &XorName) -> Result { - let mut wtxn = match env.write_txn() { - Ok(wtxn) => wtxn, - Err(heed::Error::Mdb(MdbError::MapFull)) => return Ok(DeleteOutcome::MapFull), - Err(e) => return Err(Error::Storage(format!("Failed to create write txn: {e}"))), - }; - let existed = match db.delete(&mut wtxn, key) { - Ok(existed) => existed, - Err(heed::Error::Mdb(MdbError::MapFull)) => return Ok(DeleteOutcome::MapFull), - Err(e) => return Err(Error::Storage(format!("Failed to delete chunk: {e}"))), - }; - match wtxn.commit() { - Ok(()) => Ok(DeleteOutcome::Done(existed)), - Err(heed::Error::Mdb(MdbError::MapFull)) => Ok(DeleteOutcome::MapFull), - Err(e) => Err(Error::Storage(format!("Failed to commit delete: {e}"))), - } -} - -/// Outcome of one delete attempt. -enum DeleteOutcome { - /// The delete committed; the flag is whether the key had existed. - Done(bool), - /// The map ceiling left no room for the delete's copy-on-write. - MapFull, -} - -/// Compute the LMDB map size from the disk hosting `db_dir`. -/// -/// The result covers **all existing data** plus all remaining usable disk -/// space: -/// -/// ```text -/// map_size = current_db_file_size + max(0, available_space − reserve) -/// ``` -/// -/// `available_space` (from `statvfs`) reports only the *free* bytes on the -/// partition — the DB file's own footprint is **not** included, so adding -/// it back ensures the map is always large enough for the data already -/// stored. -/// -/// On Windows the disk-headroom term is additionally capped at -/// `WINDOWS_MAP_HEADROOM` to bound the map-proportional commit overhead (see -/// that constant); [`LmdbStorage::try_resize`] extends the map on demand as -/// data grows. -/// -/// The result is page-aligned and never falls below [`MIN_MAP_SIZE`]. -fn compute_map_size(db_dir: &Path, reserve: u64) -> Result { - let available = fs2::available_space(db_dir) - .map_err(|e| Error::Storage(format!("Failed to query available disk space: {e}")))?; - - // The MDB data file may not exist yet on first run, and that is the only reason to - // read zero here. Any other failure is a question that was not answered, and answering - // it with zero sizes the map as though the database were empty, which on a node with a - // large one is a map far too small to open it. - let mdb_file = db_dir.join("data.mdb"); - let current_db_bytes = match std::fs::metadata(&mdb_file) { - Ok(meta) => meta.len(), - Err(e) if e.kind() == std::io::ErrorKind::NotFound => 0, - Err(e) => { - return Err(Error::Storage(format!( - "Failed to measure {}: {e}. Refusing to size the map as though it were \ - empty.", - mdb_file.display() - ))) - } - }; - - let target = map_target_bytes(current_db_bytes, available, reserve); - - // Align up to system page size (required by heed's resize). - let page = page_size::get() as u64; - let aligned = target.div_ceil(page) * page; - - let result = usize::try_from(aligned).unwrap_or(usize::MAX); - Ok(result.max(MIN_MAP_SIZE)) -} - -/// Head-room policy for the LMDB map, split out from [`compute_map_size`] so it -/// is unit-testable without touching the real filesystem. -/// -/// `map = current_db_bytes + max(0, available − reserve)`, with the head-room -/// term capped at `WINDOWS_MAP_HEADROOM` on Windows. Existing data -/// (`current_db_bytes`) is always covered so a resize can never truncate the -/// database, even when the head-room cap or a nearly-full disk drives the -/// growth term to zero. -fn map_target_bytes(current_db_bytes: u64, available: u64, reserve: u64) -> u64 { - // available_space excludes the DB file, so we add it back to get the - // total space the DB could occupy while still leaving `reserve` free. - let growth_room = available.saturating_sub(reserve); - - // On Windows, bound the head-room so the mapped size (and its - // size-proportional commit overhead) stays proportional to stored data - // rather than disk capacity. Elsewhere, use all reachable space. - #[cfg(windows)] - let growth_room = growth_room.min(WINDOWS_MAP_HEADROOM); - - current_db_bytes.saturating_add(growth_room) -} - -/// Map the result of a space query onto the *disk half* of the capacity -/// verdict. -/// -/// `Full` here means "below the reserve", which since ant-node #210 is only -/// half the refusal predicate: [`LmdbStorage::capacity_verdict`] goes on to -/// consult the store's reusable pages before refusing. -/// -/// Split out from [`LmdbStorage::capacity_verdict`] because the three-way -/// mapping is the part worth proving, and proving it through the filesystem is -/// not portable: asking for the free space of a directory that does not exist -/// fails on Unix but succeeds on Windows, which resolves it to the volume. -fn verdict_from_available_space(available: std::io::Result, reserve: u64) -> CapacityVerdict { - match available { - Ok(available) if available < reserve => CapacityVerdict::Full, - Ok(_) => CapacityVerdict::Writable, - Err(e) => { - warn!("Could not query available disk space: {e}"); - CapacityVerdict::Unknown - } - } -} - -/// What a capacity check concluded, when the caller needs to tell "this node is -/// full" apart from "this node could not find out". -/// -/// [`LmdbStorage::check_capacity`] collapses both into `Err`, which is right for -/// a caller that only wants to know whether to attempt a write. A caller -/// deciding *how long to stand down* needs the distinction: a full disk is a -/// standing condition worth waiting minutes on, while a failed `statvfs` may -/// have cleared by the next attempt and must not be treated as one. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum CapacityVerdict { - /// Available space is at or above the configured reserve. That is what the - /// query establishes, and possibly from the TTL cache — not a promise the - /// next write succeeds. - Writable, - /// Available space is below the configured reserve. - Full, - /// The query itself failed, so nothing is known about available space. - Unknown, -} - -#[cfg(test)] -#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] -mod tests { - use super::*; - use crate::ant_protocol::MAX_CHUNK_SIZE; - - /// Short probe used to prove `wait_idle` is still blocked on a parked op. - const WAIT_IDLE_BLOCKED_PROBE: std::time::Duration = std::time::Duration::from_millis(200); - /// Generous ceiling for `wait_idle` to complete once the op is released. - const WAIT_IDLE_COMPLETE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); - /// Poll interval while waiting for a parked raw read to take the shared lock. - const RAW_READ_LOCK_POLL: std::time::Duration = std::time::Duration::from_millis(5); - /// Ceiling for a parked raw read to take the shared lock. - const RAW_READ_LOCK_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); - /// Short probe used to prove `try_resize` is still blocked on the shared lock. - const RESIZE_BLOCKED_PROBE: std::time::Duration = std::time::Duration::from_millis(200); - /// Generous ceiling for `try_resize` to complete once the raw read releases. - const RESIZE_COMPLETE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); - - #[test] - fn map_target_covers_existing_data_and_headroom() { - // A partition with far more free space than any node needs. - let huge_free = 100 * 1024 * GIB; // 100 TiB - let reserve = 500 * MIB; - - // Fresh node, no data yet. - let fresh = map_target_bytes(0, huge_free, reserve); - // Node already holding 16 GiB of chunks. - let with_data = map_target_bytes(16 * GIB, huge_free, reserve); - - #[cfg(windows)] - { - // Windows: head-room is capped, so the map (and thus page tables) - // stay bounded regardless of disk size. Existing data always sits - // on top of the capped head-room. - assert_eq!(fresh, WINDOWS_MAP_HEADROOM); - assert_eq!(with_data, 16 * GIB + WINDOWS_MAP_HEADROOM); - // Sanity: page-table cost (~map/512) is tens of MiB, not tens of GiB. - assert!(with_data / 512 < 128 * MIB); - } - - #[cfg(not(windows))] - { - // Other platforms keep the disk-sized map (lazy page tables cost - // nothing), so head-room is the full free span minus reserve. - assert_eq!(fresh, huge_free - reserve); - assert_eq!(with_data, 16 * GIB + (huge_free - reserve)); - } - } - - #[test] - fn map_target_never_truncates_data_when_disk_nearly_full() { - // Free space below the reserve → head-room saturates to 0 on every - // platform, but the existing 4 GiB of data must still be covered. - assert_eq!(map_target_bytes(4 * GIB, 100 * MIB, 500 * MIB), 4 * GIB); - } - - /// Regression (V2-620 review): `all_keys` and `get_raw` must take the shared - /// `env_lock` so their LMDB read transaction can never run concurrently with - /// `try_resize()`'s unsafe `Env::resize()` — which this PR turns into a - /// routine ~per-32-GiB Windows event. We prove it by holding the exclusive - /// lock (as a resize does) and asserting both calls block until it is freed. - /// - /// Holding a `parking_lot` guard across `.await` is deliberate and safe here: - /// the guard stays on this current-thread test task while `all_keys`/`get_raw` - /// run their blocking work on the `spawn_blocking` pool (separate threads). - #[tokio::test] - #[allow(clippy::await_holding_lock)] - async fn read_paths_block_while_env_is_being_resized() { - use std::time::Duration; - let (storage, _temp) = create_test_storage().await; - - // Store one chunk so the read paths have real work to return. - let content = b"resize-safety"; - let address = LmdbStorage::compute_address(content); - storage.put(&address, content).await.expect("put"); - - // Simulate a resize in progress: hold the exclusive env lock. - let write_guard = storage.env_lock.write(); - - // Neither read path may complete while the exclusive lock is held — - // before the fix they took no lock and would return immediately. - assert!( - tokio::time::timeout(Duration::from_millis(250), storage.all_keys()) - .await - .is_err(), - "all_keys completed while env_lock was held exclusively — missing shared guard" - ); - assert!( - tokio::time::timeout(Duration::from_millis(250), storage.get_raw(&address)) - .await - .is_err(), - "get_raw completed while env_lock was held exclusively — missing shared guard" - ); - - // Once the exclusive lock is released, both proceed and see the data. - drop(write_guard); - assert_eq!(storage.all_keys().await.expect("all_keys").len(), 1); - assert_eq!( - storage.get_raw(&address).await.expect("get_raw").as_deref(), - Some(content.as_slice()) - ); - } - - async fn create_test_storage() -> (LmdbStorage, tempfile::TempDir) { - let temp_dir = tempfile::TempDir::new().expect("create temp dir"); - let config = LmdbStorageConfig { - root_dir: temp_dir.path().to_path_buf(), - ..LmdbStorageConfig::test_default() - }; - let storage = LmdbStorage::new(config).await.expect("create storage"); - (storage, temp_dir) - } - - #[tokio::test] - async fn test_put_and_get() { - let (storage, _temp) = create_test_storage().await; - - let content = b"hello world"; - let address = LmdbStorage::compute_address(content); - - // Store chunk - let is_new = storage.put(&address, content).await.expect("put"); - assert!(is_new); - - // Retrieve chunk - let retrieved = storage.get(&address).await.expect("get"); - assert_eq!(retrieved, Some(content.to_vec())); - } - - #[tokio::test] - async fn test_put_duplicate() { - let (storage, _temp) = create_test_storage().await; - - let content = b"test data"; - let address = LmdbStorage::compute_address(content); - - // First store - let is_new1 = storage.put(&address, content).await.expect("put 1"); - assert!(is_new1); - - // Duplicate store - let is_new2 = storage.put(&address, content).await.expect("put 2"); - assert!(!is_new2); - - // Check stats - let stats = storage.stats(); - assert_eq!(stats.chunks_stored, 1); - assert_eq!(stats.duplicates, 1); - } - - #[tokio::test] - async fn test_get_not_found() { - let (storage, _temp) = create_test_storage().await; - - let address = [0xAB; 32]; - let result = storage.get(&address).await.expect("get"); - assert!(result.is_none()); - } - - #[tokio::test] - async fn test_exists() { - let (storage, _temp) = create_test_storage().await; - - let content = b"exists test"; - let address = LmdbStorage::compute_address(content); - - assert!(!storage.exists(&address).expect("exists")); - - storage.put(&address, content).await.expect("put"); - - assert!(storage.exists(&address).expect("exists")); - } - - #[tokio::test] - async fn test_delete() { - let (storage, _temp) = create_test_storage().await; - - let content = b"delete test"; - let address = LmdbStorage::compute_address(content); - - // Store - storage.put(&address, content).await.expect("put"); - assert!(storage.exists(&address).expect("exists")); - - // Delete - let deleted = storage.delete(&address).await.expect("delete"); - assert!(deleted); - assert!(!storage.exists(&address).expect("exists")); - - // Delete again (already deleted) - let deleted2 = storage.delete(&address).await.expect("delete 2"); - assert!(!deleted2); - } - - #[tokio::test] - async fn test_address_mismatch() { - let (storage, _temp) = create_test_storage().await; - - let content = b"some content"; - let wrong_address = [0xFF; 32]; // Wrong address - - let result = storage.put(&wrong_address, content).await; - assert!(result.is_err()); - assert!(result.unwrap_err().to_string().contains("mismatch")); - } - - #[test] - fn test_compute_address() { - // Known BLAKE3 hash of "hello world" - let content = b"hello world"; - let address = LmdbStorage::compute_address(content); - - let expected_hex = "d74981efa70a0c880b8d8c1985d075dbcbf679b99a5f9914e5aaf96b831a9e24"; - assert_eq!(hex::encode(address), expected_hex); - } - - #[tokio::test] - async fn test_stats() { - let (storage, _temp) = create_test_storage().await; - - let content1 = b"content 1"; - let content2 = b"content 2"; - let address1 = LmdbStorage::compute_address(content1); - let address2 = LmdbStorage::compute_address(content2); - - // Store two chunks - storage.put(&address1, content1).await.expect("put 1"); - storage.put(&address2, content2).await.expect("put 2"); - - // Retrieve one - storage.get(&address1).await.expect("get"); - - let stats = storage.stats(); - assert_eq!(stats.chunks_stored, 2); - assert_eq!(stats.chunks_retrieved, 1); - assert_eq!( - stats.bytes_stored, - content1.len() as u64 + content2.len() as u64 - ); - assert_eq!(stats.bytes_retrieved, content1.len() as u64); - assert_eq!(stats.current_chunks, 2); - } - - #[tokio::test] - async fn test_persistence_across_reopen() { - let temp_dir = tempfile::TempDir::new().expect("create temp dir"); - let content = b"persistent data"; - let address = LmdbStorage::compute_address(content); - - // Store a chunk - { - let config = LmdbStorageConfig { - root_dir: temp_dir.path().to_path_buf(), - ..LmdbStorageConfig::test_default() - }; - let storage = LmdbStorage::new(config).await.expect("create storage"); - storage.put(&address, content).await.expect("put"); - } - - // Re-open and verify it persisted - { - let config = LmdbStorageConfig { - root_dir: temp_dir.path().to_path_buf(), - ..LmdbStorageConfig::test_default() - }; - let storage = LmdbStorage::new(config).await.expect("reopen storage"); - assert_eq!(storage.current_chunks().expect("current_chunks"), 1); - let retrieved = storage.get(&address).await.expect("get"); - assert_eq!(retrieved, Some(content.to_vec())); - } - } - - #[tokio::test] - async fn test_all_keys() { - let (storage, _temp) = create_test_storage().await; - - // Empty storage - let keys = storage.all_keys().await.expect("all_keys empty"); - assert!(keys.is_empty()); - - // Store some chunks - let content1 = b"chunk one for keys"; - let content2 = b"chunk two for keys"; - let addr1 = LmdbStorage::compute_address(content1); - let addr2 = LmdbStorage::compute_address(content2); - storage.put(&addr1, content1).await.expect("put 1"); - storage.put(&addr2, content2).await.expect("put 2"); - - let mut keys = storage.all_keys().await.expect("all_keys"); - keys.sort_unstable(); - let mut expected = vec![addr1, addr2]; - expected.sort_unstable(); - assert_eq!(keys, expected); - } - - #[tokio::test] - async fn test_get_raw() { - let (storage, _temp) = create_test_storage().await; - - let content = b"raw test data"; - let address = LmdbStorage::compute_address(content); - storage.put(&address, content).await.expect("put"); - - // get_raw returns bytes without verification - let raw = storage.get_raw(&address).await.expect("get_raw"); - assert_eq!(raw, Some(content.to_vec())); - - // Non-existent key - let missing = storage.get_raw(&[0xFF; 32]).await.expect("get_raw missing"); - assert!(missing.is_none()); - } - - /// Dropping a put's awaiter does not cancel its `spawn_blocking` LMDB - /// transaction; `wait_idle` must wait for that detached write, and the - /// storage must remain usable afterwards. - // Holding the gate's write guard across awaits is the point of the test: - // it parks the blocking closure while we probe wait_idle. - #[allow(clippy::await_holding_lock)] - #[tokio::test] - async fn wait_idle_waits_for_detached_put_blocking_op() { - let (storage, _temp) = create_test_storage().await; - - let content = b"detached put survives its dropped awaiter"; - let address = LmdbStorage::compute_address(content); - - // Park the put's blocking closure on the test gate. - let gate = storage.test_put_gate(); - let parked = gate.write(); - - // Drop the awaiting future mid-flight: the biased select! polls the - // put once — far enough to spawn the blocking task, which parks on - // the gate — then completes on the ready branch, dropping the put. - tokio::select! { - biased; - res = storage.put(&address, content) => { - panic!("put must be parked on the test gate, got {res:?}") - } - () = std::future::ready(()) => {} - } - - // The blocking op is still running: wait_idle must not complete. - let blocked = tokio::time::timeout(WAIT_IDLE_BLOCKED_PROBE, storage.wait_idle()).await; - assert!( - blocked.is_err(), - "wait_idle returned while the blocking op was parked" - ); - - // Release the gate: the detached closure commits and exits. - drop(parked); - tokio::time::timeout(WAIT_IDLE_COMPLETE_TIMEOUT, storage.wait_idle()) - .await - .expect("wait_idle after release"); - - // The dropped awaiter did not lose the write: it committed. - assert!(storage.exists(&address).expect("exists after release")); - - // The storage remains usable after wait_idle (tracker reopened). - let more = b"storage still usable after wait_idle"; - let more_addr = LmdbStorage::compute_address(more); - assert!(storage - .put(&more_addr, more) - .await - .expect("put after wait_idle")); - } - - /// A map resize takes the environment's *exclusive* lock, so it must wait - /// for in-flight raw reads (which hold the *shared* lock) to finish before - /// calling `env.resize()`. This proves `get_raw` holds that shared lock for - /// the whole duration of its blocking closure; `all_keys` uses the same - /// guard. - // Holding the gate's write guard across awaits is the point of the test: - // it parks the raw read's blocking closure while we probe try_resize. - #[allow(clippy::await_holding_lock)] - #[tokio::test] - async fn resize_waits_for_in_flight_raw_read() { - let (storage, _temp) = create_test_storage().await; - - let content = b"raw read holds the shared env lock"; - let address = LmdbStorage::compute_address(content); - storage.put(&address, content).await.expect("put"); - - // Park the raw read's blocking closure on the test gate. It acquires - // the shared env_lock first, then parks here still holding it. - let gate = storage.test_read_gate(); - let parked = gate.write(); - - // Drop the awaiting future mid-flight: the biased select! polls get_raw - // once — far enough to spawn the blocking task, which takes the shared - // lock and parks on the gate — then completes on the ready branch, - // dropping the awaiter. The detached closure keeps holding the lock. - tokio::select! { - biased; - res = storage.get_raw(&address) => { - panic!("get_raw must be parked on the test gate, got {res:?}") - } - () = std::future::ready(()) => {} - } - - // Wait until the detached read has actually taken the shared lock, - // signalled by the exclusive half no longer being immediately available. - tokio::time::timeout(RAW_READ_LOCK_TIMEOUT, async { - loop { - let free = storage.env_lock.try_write().is_some(); - if !free { - break; - } - tokio::time::sleep(RAW_READ_LOCK_POLL).await; - } - }) - .await - .expect("raw read did not take the shared env lock"); - - // A resize needs the exclusive lock, so it must block while the raw - // read holds the shared lock. - let resize = storage.try_resize(); - tokio::pin!(resize); - let blocked = tokio::time::timeout(RESIZE_BLOCKED_PROBE, &mut resize).await; - assert!( - blocked.is_err(), - "try_resize completed while a raw read held the shared env lock" - ); - - // Release the read: it drops the shared lock, letting the resize take - // the exclusive lock and finish. - drop(parked); - tokio::time::timeout(RESIZE_COMPLETE_TIMEOUT, &mut resize) - .await - .expect("try_resize did not complete after the raw read released") - .expect("try_resize"); - } - - /// The gate fires on `Full` and only on `Full`, so the verdict has to tell a - /// disk below its reserve from one above it, and has to notice when that - /// stops being true. - /// - /// The third call is the one that matters for recovery: the below-reserve - /// condition clears, and the verdict has to follow it rather than stay stuck - /// on its earlier answer. A node that remembered a refusal would stop - /// fetching for good. - /// - /// What this does not show: that a changed free-space reading is re-read from - /// the filesystem. The condition is cleared by dropping the reserve, which is - /// the same comparison approached from the other side. - #[tokio::test] - async fn capacity_verdict_follows_the_reserve_and_notices_recovery() { - let (writable, _temp) = create_test_storage().await; - assert_eq!(writable.capacity_verdict(), CapacityVerdict::Writable); - - let temp_dir = tempfile::TempDir::new().expect("create temp dir"); - let config = LmdbStorageConfig { - root_dir: temp_dir.path().to_path_buf(), - // Far above any real free space, the same way the e2e builds a - // write-blocked node. - disk_reserve: u64::MAX / 2, - ..LmdbStorageConfig::test_default() - }; - let mut full = LmdbStorage::new(config).await.expect("create storage"); - assert_eq!(full.capacity_verdict(), CapacityVerdict::Full); - - // Still short of space, so still `Full`. A `Full` result that populated - // the passing-result cache would answer `Writable` here. - assert_eq!(full.capacity_verdict(), CapacityVerdict::Full); - - // Space is no longer short. Nothing cached a refusal, so the very next - // read has to see it. - full.config.disk_reserve = 0; - assert_eq!( - full.capacity_verdict(), - CapacityVerdict::Writable, - "a refusal must not be negatively cached: the next read has to see the \ - below-reserve condition clear" - ); - } - - /// A space query that fails says nothing about available space, so it must - /// not read as a full disk. The gate stands a key down for five minutes on - /// `Full` alone, and a failed `statvfs` is not a condition worth standing - /// down for: it may be gone by the next cycle. - /// - /// The two `Ok` cases pin the boundary the reserve names: equal to the - /// reserve is writable, one byte under it is not. - /// - /// What this does not show: that the gate leaves `Unknown` alone. The gate - /// tests `== Full` on a separate line inside the verification cycle, which - /// needs a network to reach, so this covers the classification only. - #[test] - fn a_failed_space_query_reads_as_unknown_not_full() { - const RESERVE: u64 = 1024; - - assert_eq!( - verdict_from_available_space(Err(std::io::Error::other("space query failed")), RESERVE), - CapacityVerdict::Unknown, - "a failed query must not be reported as a full disk" - ); - - assert_eq!( - verdict_from_available_space(Ok(RESERVE - 1), RESERVE), - CapacityVerdict::Full - ); - assert_eq!( - verdict_from_available_space(Ok(RESERVE), RESERVE), - CapacityVerdict::Writable, - "at the reserve is not below it" - ); - } - - /// The verdict and the pre-check have to refuse on the same condition. - /// - /// They are two readings of one question — can this node write? — and two - /// callers depend on them separately: the verification cycle gates the - /// close-group probe on the verdict, while `execute_single_fetch` gates the - /// dial on the pre-check. A verdict stricter than the pre-check is the - /// harmful direction. A node the pre-check would let write stops - /// discovering holders for keys it could have stored, which is - /// under-replication rather than a saved probe, and nothing else in the - /// change would notice. - /// - /// This is a tripwire, deliberately built on the state where the two are - /// about to part company rather than on a bare full disk. ant-node - /// \#210 makes the pre-check a two-part predicate — below the reserve *and* - /// out of reusable pages inside the store — and a store that has deleted - /// more than one chunk's worth of pages fails only the first half, because - /// LMDB returns those pages to its own free list and never to the - /// filesystem. On a bare full disk the two predicates still agree, so a - /// test built on one would pass straight through the divergence. Whichever - /// of the two changes merges second has to carry the second half into the - /// verdict, and this is what makes that a red test rather than a textual - /// conflict resolved without it. - /// - /// What this does not show: which of `Full` and `Unknown` a refusal is. - /// `a_failed_space_query_reads_as_unknown_not_full` pins that, and only - /// `Full` reaches the gate. - #[tokio::test] - async fn capacity_verdict_refuses_exactly_when_check_capacity_does() { - let (mut storage, _temp) = create_test_storage().await; - assert_eq!(storage.capacity_verdict(), CapacityVerdict::Writable); - assert!( - storage.check_capacity().is_ok(), - "a writable verdict has to mean the pre-check admits the write" - ); - - // Two chunks written and deleted, so the store sits on more than one - // chunk of space LMDB can reuse and the filesystem will never take - // back. This is the pruned node the two predicates disagree about. - for fill in [1u8, 2u8] { - let content = vec![fill; MAX_CHUNK_SIZE]; - let address = LmdbStorage::compute_address(&content); - assert!(storage.put(&address, &content).await.expect("put")); - assert!(storage.delete(&address).await.expect("delete")); - } - - // Established without either function under test, so the scenario does - // not rest on the thing being measured. - storage.config.disk_reserve = u64::MAX / 2; - *storage.last_disk_ok.lock() = None; - let available = fs2::available_space(&storage.env_dir).expect("query available space"); - assert!( - available < storage.config.disk_reserve, - "the volume has to read as below the reserve, or neither predicate is \ - being asked the interesting question" - ); - // Reusable bytes read the way the two-part predicate reads them: the - // file's size less the pages still holding data. `Env::stat` rather - // than `non_free_pages_size`, which calls `String::from_utf8(key)` and - // unwraps, so it panics on our 32 random bytes of key. - let stat = storage.env.stat(); - let live_pages = (stat.branch_pages as u64) - .saturating_add(stat.leaf_pages as u64) - .saturating_add(stat.overflow_pages as u64); - let live_bytes = live_pages.saturating_mul(u64::from(stat.page_size)); - let file_bytes = storage.env.real_disk_size().expect("query store file size"); - let reusable = file_bytes.saturating_sub(live_bytes); - assert!( - reusable > MAX_CHUNK_SIZE as u64, - "the store has to sit on more than one chunk of reusable space, or the \ - two predicates are not yet being asked to differ \ - (file_bytes={file_bytes}, live_bytes={live_bytes})" - ); - - // Neither call caches a refusal, so the order of the two does not - // decide either answer. - let pre_check_refuses = storage.check_capacity().is_err(); - let verdict = storage.capacity_verdict(); - assert_eq!( - pre_check_refuses, - verdict != CapacityVerdict::Writable, - "the dial pre-check and the verification gate disagree about whether \ - this node can write: check_capacity refuses={pre_check_refuses}, \ - verdict={verdict:?}. A verdict of Full under a pre-check that admits \ - the write leaves a node that can store chunks refusing to look for \ - them" - ); - } - - // ── Capacity below the disk reserve (LMDB reuse) ──────────────────── - - /// Value size for the reuse tests. A whole number of chunks' worth, so the - /// space freed by a few deletes is unambiguously enough for one more. - const REUSE_VALUE_LEN: usize = 1024 * 1024; - - /// Distinct filler of `REUSE_VALUE_LEN` bytes. - fn reuse_filler(seed: u32) -> Vec { - let mut content = seed.to_le_bytes().to_vec(); - content.resize(REUSE_VALUE_LEN, 0u8); - content - } - - /// A config for `dir` whose reserve exceeds any real disk, so the store - /// always sees itself as below the reserve. - fn below_reserve_config(dir: &Path) -> LmdbStorageConfig { - LmdbStorageConfig { - root_dir: dir.to_path_buf(), - disk_reserve: u64::MAX, - ..LmdbStorageConfig::test_default() - } - } - - /// Write `count` chunks with an unconstrained reserve, returning their - /// addresses in insertion order. - async fn seed_chunks(dir: &Path, count: u32) -> Vec { - let config = LmdbStorageConfig { - root_dir: dir.to_path_buf(), - ..LmdbStorageConfig::test_default() - }; - let storage = LmdbStorage::new(config).await.expect("create storage"); - - let mut addresses = Vec::new(); - for seed in 0..count { - let content = reuse_filler(seed); - let address = LmdbStorage::compute_address(&content); - storage.put(&address, &content).await.expect("seed put"); - addresses.push(address); - } - - storage.wait_idle().await; - addresses - } - - fn file_len(storage: &LmdbStorage) -> u64 { - storage.env.real_disk_size().expect("real_disk_size") - } - - /// The regression this change is about: a node whose volume is below the - /// reserve must still write into pages an earlier delete freed. Before the - /// fix the pre-check refused on the disk half alone, so a node that had - /// pruned sat on reusable space it could not use. - #[tokio::test] - async fn below_reserve_put_reuses_freed_pages() { - let temp_dir = tempfile::TempDir::new().expect("create temp dir"); - let seeded = seed_chunks(temp_dir.path(), 12).await; - - let storage = LmdbStorage::new(below_reserve_config(temp_dir.path())) - .await - .expect("reopen storage"); - - let content = reuse_filler(u32::MAX); - let address = LmdbStorage::compute_address(&content); - - // Nothing freed yet, so this write would have to grow the file. Below - // the reserve that is exactly what must be refused. - assert!( - storage.put(&address, &content).await.is_err(), - "a write that must grow the file was allowed below the reserve" - ); - - // Free several chunks. Their pages go on LMDB's free list, not back to - // the filesystem, so `statvfs` still reports the volume as full. - for seeded_address in seeded.iter().take(6) { - assert!(storage.delete(seeded_address).await.expect("delete")); - } - - let before = file_len(&storage); - let stored = storage - .put(&address, &content) - .await - .expect("put into freed pages was refused below the reserve"); - assert!(stored); - assert_eq!( - storage.get(&address).await.expect("get"), - Some(content), - "chunk written into reused pages did not read back" - ); - - // The whole point: it was served from inside the existing file. - assert_eq!( - file_len(&storage), - before, - "reusing freed pages grew data.mdb, consuming the reserve" - ); - } - - /// A refused write must not have grown the file on its way to failing, - /// which is what protects the reserve while the map is pinned. - #[tokio::test] - async fn below_reserve_refused_put_does_not_grow_the_file() { - let temp_dir = tempfile::TempDir::new().expect("create temp dir"); - let _ = seed_chunks(temp_dir.path(), 6).await; - - let storage = LmdbStorage::new(below_reserve_config(temp_dir.path())) - .await - .expect("reopen storage"); - - let content = reuse_filler(u32::MAX); - let address = LmdbStorage::compute_address(&content); - - // Take the baseline after the first attempt, so it includes the pin. - let refusal = storage - .put(&address, &content) - .await - .expect_err("a write that must grow the file was allowed"); - assert!( - refusal.to_string().contains("Insufficient disk space"), - "refused for the wrong reason: {refusal}" - ); - let before = file_len(&storage); - let pinned_map = storage.env.info().map_size; - - for seed in 0..4u32 { - let content = reuse_filler(u32::MAX - 1 - seed); - let address = LmdbStorage::compute_address(&content); - let refusal = storage - .put(&address, &content) - .await - .expect_err("a write that must grow the file was allowed"); - assert!( - refusal.to_string().contains("Insufficient disk space"), - "refused for the wrong reason: {refusal}" - ); - } - - assert_eq!( - storage.env.info().map_size, - pinned_map, - "the map ceiling drifted while writes were being refused" - ); - - assert_eq!( - file_len(&storage), - before, - "refused writes still extended data.mdb into the reserve" - ); - } - - /// One refused maximum-sized value must not lock out smaller ones. LMDB's - /// `MapFull` is specific to the allocation it was asked for, so remembering - /// it store-wide would let a single large chunk deny every later write. - #[tokio::test] - async fn large_refusal_does_not_block_a_smaller_put() { - let temp_dir = tempfile::TempDir::new().expect("create temp dir"); - let seeded = seed_chunks(temp_dir.path(), 10).await; - - let storage = LmdbStorage::new(below_reserve_config(temp_dir.path())) - .await - .expect("reopen storage"); - - // Free room for a small value, but not for a large one. - let Some(first) = seeded.first() else { - panic!("seed_chunks returned no addresses"); - }; - assert!(storage.delete(first).await.expect("delete")); - - // A value far larger than what was freed cannot fit. - let oversized = vec![3u8; 8 * REUSE_VALUE_LEN]; - let oversized_address = LmdbStorage::compute_address(&oversized); - assert!(storage.put(&oversized_address, &oversized).await.is_err()); - - // A small value still must, using the pages the delete released. - let small = b"small record that fits in a freed page".to_vec(); - let small_address = LmdbStorage::compute_address(&small); - let stored = storage - .put(&small_address, &small) - .await - .expect("a large refusal blocked a small put that had room"); - assert!(stored); - } - - /// A store with no reusable page must still be able to delete, or it can - /// never prune its way back to health. - #[tokio::test] - async fn full_store_below_reserve_can_still_delete() { - let temp_dir = tempfile::TempDir::new().expect("create temp dir"); - let seeded = seed_chunks(temp_dir.path(), 8).await; - - let storage = LmdbStorage::new(below_reserve_config(temp_dir.path())) - .await - .expect("reopen storage"); - - // Pin the map by attempting a write that cannot fit. - let content = reuse_filler(u32::MAX); - let address = LmdbStorage::compute_address(&content); - assert!(storage.put(&address, &content).await.is_err()); - assert!(storage.no_growth.load(Ordering::Acquire)); - - let pinned_map = storage.env.info().map_size; - - for seeded_address in &seeded { - assert!( - storage.delete(seeded_address).await.expect("delete"), - "a pinned store could not prune" - ); - } - assert_eq!(storage.current_chunks().expect("current_chunks"), 0); - - // The allowance bounds permanent file growth, not the number of - // assisted deletes. Pruning a pinned store must stay possible however - // many deletes it takes, so whatever was charged has to be growth the - // file really took, and has to stay inside the budget. - let charged = storage.delete_growth_charged.load(Ordering::Acquire); - assert!( - charged < DELETE_COW_GROWTH_BUDGET, - "deletes exhausted the growth allowance ({charged} B) while pruning a pinned store" - ); - - // Whether or not any delete needed the maintenance allowance, none of - // it may be left in the ceiling afterwards: a raised ceiling is - // ordinary put capacity, so leaking it hands away the reserve. - let file_bytes = file_len(&storage); - assert!( - storage.env.info().map_size as u64 <= file_bytes.max(pinned_map as u64), - "delete left maintenance slack in the map ceiling" - ); - - // And the store must still refuse a write it cannot fit, i.e. the pin - // is still doing its job after the prune. - let oversized = vec![9u8; 64 * REUSE_VALUE_LEN]; - let oversized_address = LmdbStorage::compute_address(&oversized); - assert!( - storage.put(&oversized_address, &oversized).await.is_err(), - "pinning stopped working after a delete used the allowance" - ); - } - - /// The pre-check must admit while reuse is plausible and refuse once it is - /// not. Refusing on `statvfs` alone is what blinded a node to its own free - /// pages, so being below the reserve cannot by itself be an error. - #[tokio::test] - async fn check_capacity_tracks_reusable_space_not_just_disk() { - let temp_dir = tempfile::TempDir::new().expect("create temp dir"); - let seeded = seed_chunks(temp_dir.path(), 16).await; - - let storage = LmdbStorage::new(below_reserve_config(temp_dir.path())) - .await - .expect("reopen storage"); - - // A freshly written store has almost no free page, so below the reserve - // the pre-check refuses and the caller skips its expensive setup. - assert!( - storage.check_capacity().is_err(), - "pre-check stayed open on a store with no reusable space" - ); - - // Pruning puts pages back on the free list. Nothing is returned to the - // filesystem, so `statvfs` is unchanged and only the reusable half of - // the predicate can reopen the node. - for seeded_address in seeded.iter().take(10) { - assert!(storage.delete(seeded_address).await.expect("delete")); - } - - storage - .check_capacity() - .expect("pre-check stayed closed after pruning freed pages"); - } - - /// Freeing disk must lift the pin, or a node would stay clamped to its - /// high-water mark after an operator grew the partition. - #[tokio::test] - async fn leaving_no_growth_restores_head_room() { - let temp_dir = tempfile::TempDir::new().expect("create temp dir"); - let _ = seed_chunks(temp_dir.path(), 6).await; - - let mut storage = LmdbStorage::new(below_reserve_config(temp_dir.path())) - .await - .expect("reopen storage"); - - let content = reuse_filler(u32::MAX); - let address = LmdbStorage::compute_address(&content); - assert!(storage.put(&address, &content).await.is_err()); - assert!(storage.no_growth.load(Ordering::Acquire)); - let pinned_map = storage.env.info().map_size; - - // Simulate the operator freeing space: the reserve is now satisfiable. - storage.config.disk_reserve = 0; - *storage.last_disk_ok.lock() = None; - - let stored = storage - .put(&address, &content) - .await - .expect("store stayed pinned after disk was freed"); - assert!(stored); - assert!(!storage.no_growth.load(Ordering::Acquire)); - assert!( - storage.env.info().map_size > pinned_map, - "map was not re-grown after leaving no-growth mode" - ); - } - - /// Above the reserve nothing changes: no pinning, and writes grow the file - /// on demand exactly as before. - #[tokio::test] - async fn above_reserve_behaviour_is_unchanged() { - let (storage, _temp) = create_test_storage().await; - - storage - .check_capacity() - .expect("pre-check on a healthy node"); - - let content = reuse_filler(1); - let address = LmdbStorage::compute_address(&content); - assert!(storage.put(&address, &content).await.expect("put")); - assert!(!storage.no_growth.load(Ordering::Acquire)); - storage - .check_capacity() - .expect("pre-check after a healthy put"); - } -} diff --git a/src/storage/migration.rs b/src/storage/migration.rs deleted file mode 100644 index 3087941d..00000000 --- a/src/storage/migration.rs +++ /dev/null @@ -1,3142 +0,0 @@ -//! Moving a node off LMDB and onto the file store without losing a chunk. -//! -//! LMDB never returns a deleted page to the filesystem. Disk comes back exactly once, -//! when `chunks.mdb` is removed whole, so a node cannot free space by deleting chunks -//! and cannot compact its way out either (compaction needs free space equal to the live -//! data, which is the same condition). That single fact shapes everything here. -//! -//! # The two ways this could lose data, and why neither can happen -//! -//! 1. **A node deletes its LMDB before the chunks are safely in files.** Retirement is -//! gated on the file store already holding every key the node still claims, and on -//! the existing retention contract: a key the node is still answerable for under a -//! gossiped commitment vetoes the delete. -//! 2. **Every node sheds the same chunk at once.** A node only sheds when it cannot fit -//! its own payload, and it sheds by close-group rank, furthest first. A chunk has -//! exactly one 7th-closest and one 6th-closest holder, so it is only ever a shed -//! candidate for two of its seven holders, and the staged rollout brings that to one. -//! -//! # Three releases -//! -//! Slashing is the *auditor's* decision, so a node cannot protect itself from being -//! penalised for a shed. Everyone else has to stop first, which is why this lands over -//! three releases rather than one: -//! -//! | Release | Penalise not holding a close-group chunk? | [`MigrationConfig::retire_legacy`] | -//! |---|---|---| -//! | First: stop that one penalty | no | `false` | -//! | Second: migrate | no | `true` | -//! | Third: restore it | yes | `true` | -//! -//! The penalty column is not a field here. It lives once, in -//! [`crate::replication::config::RELEASE_SUSPEND_CLOSE_GROUP_STORAGE_PENALTY`], and both -//! the auditors that apply it and the shedder that depends on it being withheld read that -//! same switch. Two copies would let a node shed while its peers still penalised. -//! -//! Audits keep running and keep recording throughout. What the first release withholds is narrow and -//! deliberate: only the penalty for *not holding a close-group chunk*. The -//! commitment-bound subtree audit still penalises in every release, because the whole -//! migration turns on a node's reduced commitment still being binding. The record those -//! audits keep is also how we will know when the third release is safe to ship. - -use crate::ant_protocol::XorName; -use crate::error::{Error, Result}; -use crate::logging::{debug, info, warn}; -use crate::replication::config::{storage_admission_width, ReplicationConfig}; -use crate::replication::pruning::{ - prove_peers_hold_records, prune_proofs_needed, target_peers_reported_present, -}; -use crate::storage::chunk_store::{ChunkStore, VerifyReport}; -use saorsa_core::identity::PeerId; -use saorsa_core::P2PNode; -use serde::{Deserialize, Serialize}; -use std::collections::{HashMap, HashSet}; -use std::path::{Path, PathBuf}; -use std::sync::Arc; -use std::time::{Duration, Instant}; -use std::time::{SystemTime, UNIX_EPOCH}; -use tokio_util::sync::CancellationToken; - -/// Filename of the persisted migration marker, under the node root. -pub const MIGRATION_STATE_FILE: &str = "migration-state.json"; - -/// Marker schema this build writes and understands. -const STATE_SCHEMA: u32 = 1; - -/// Floor on [`MigrationConfig::retire_delay_hours`]. -/// -/// `GOSSIP_ANSWERABILITY_TTL` is three hours at a one-hour rotation cadence, so a -/// commitment that named a shed key stops being answerable three hours after it was last -/// gossiped. Four hours clears that with an hour to spare. -pub const MIN_RETIRE_DELAY_HOURS: u64 = 4; - -/// How long between attempts to reopen an environment this node has lost its handle to. -/// -/// Each attempt scans every key in it, so retrying on every tick would spend a large store -/// entirely on failing to open it. -const HANDLE_RECOVERY_INTERVAL: Duration = Duration::from_secs(300); - -/// The longest one node may hold the volume migration lock before giving others a turn. -/// -/// Every branch that waits rather than works is meant to give the lock back on its own. -/// This is the backstop for the one that does not: without it, a node stuck on a condition -/// that never resolves stops every other node sharing the disk from ever starting, for the -/// whole release. Longer than a copy pass and a verification take, so it never interrupts -/// a node that is genuinely working. -const MAX_VOLUME_LOCK_HOLD: Duration = Duration::from_secs(6 * 3600); - -/// How long a node stands back after the cap takes the volume lock off it. -/// -/// Long enough that another node waiting on the lock actually gets it, rather than losing -/// the race to the node that has just been holding it for six hours. -const VOLUME_LOCK_COOLDOWN: Duration = Duration::from_secs(120); - -/// How many commitment rebuilds must be observed after the node commits to its -/// file-backed set before the legacy environment may be retired. -/// -/// One proves the builder read the new set. Two proves it published and survived a -/// rotation, which is what makes the retention window meaningful. -pub const REQUIRED_REBUILDS_BEFORE_RETIRE: u32 = 2; - -/// Operator-facing controls for the migration. -// Four independent switches, three of which are operator controls and one of which is a -// release constant. Collapsing them into an enum would tie choices together that are -// deliberately separate. -#[allow(clippy::struct_excessive_bools)] -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct MigrationConfig { - /// Run the background copier at all. - /// - /// Turning this off leaves a node reading the union of both stores forever. It never - /// frees the LMDB's disk, so it is an escape hatch rather than a supported mode. - #[serde(default = "default_true")] - pub enabled: bool, - - /// Write every new chunk to the legacy environment as well as the file store. - /// - /// Costs roughly a gigabyte per node per month at the observed fill rate, and buys - /// the ability to roll the fleet back: a chunk uploaded during the bridge to holders - /// that all revert to a pre-migration build would otherwise be gone from every one - /// of them. Automatically irrelevant once the legacy environment is retired. - #[serde(default = "default_true")] - pub dual_write_legacy: bool, - - /// Allow a node that cannot fit its payload to drop its furthest keys. - /// - /// An operator who would rather add disk than shed can set this to `false`. The node - /// then keeps both stores and never frees the LMDB's space. - #[serde(default = "default_true")] - pub allow_shed: bool, - - /// Delete `chunks.mdb` once the retirement gate is satisfied. - /// - /// **On in this release**, because it is the only step that returns disk. It is also - /// the only destructive step in the whole migration and the only one that cannot be - /// undone, which is why everything in front of it is a gate: the wave the node is - /// assigned to, the shed hold, the reduced commitment reaching the close group, - /// possession of every chunk being given up proven elsewhere, a re-read of every - /// remaining chunk, and a retention delay on top. - /// - /// Deliberately never serialised. A node writes its effective configuration back to - /// disk, so shipping this as an ordinary field would bake R1's `false` into every - /// operator's config file and R2 would then never retire anything. The release phase - /// belongs to the build, not to the operator's file. `ANT_MIGRATION_RETIRE_LEGACY` - /// overrides it for a canary. - #[serde(skip, default = "release_retire_legacy")] - pub retire_legacy: bool, - - /// Hours after this build first starts before a node may shed anything. - /// - /// Long enough for peers still on a pre-R1 build to upgrade, because one of those - /// still penalises a shedder at the full audit weight. - #[serde(default = "default_shed_hold_hours")] - pub shed_hold_hours: u64, - - /// Hours between committing to the file-backed key set and deleting `chunks.mdb`. - /// - /// Clamped up to [`MIN_RETIRE_DELAY_HOURS`]. Longer buys a rollback window on nodes - /// that can afford to hold both copies. - #[serde(default = "default_retire_delay_hours")] - pub retire_delay_hours: u64, - - /// Free megabytes the copier leaves untouched, on top of the disk reserve. - /// - /// The copier stops here rather than filling to the brink, so a node that is - /// mid-migration still has room to accept a chunk it is paid for. - #[serde(default = "default_copier_slack_mb")] - pub copier_slack_mb: u64, - - /// Copy rate ceiling, in mebibytes per second. - /// - /// The quiet responsible audit lane is where audit timeouts actually cost trust, and - /// an unthrottled copier competing with it for I/O is the fastest way to turn a - /// storage migration into an audit incident. - #[serde(default = "default_copier_throttle_mib_per_sec")] - pub copier_throttle_mib_per_sec: u64, - - /// Hours between one migration wave opening and the next. - /// - /// A close group is split into waves so that only - /// [`CONCURRENT_MIGRATIONS_PER_GROUP`] of it give chunks up at a time. This is how - /// long a wave gets to finish copying, retiring and refetching before the next one may - /// start. Only nodes that have to give something up wait for their wave; a node with - /// room migrates immediately. - #[serde(default = "default_wave_hours")] - pub wave_hours: u64, - - /// Seconds between copier ticks. - #[serde(default = "default_tick_secs")] - pub tick_secs: u64, - - /// Chunks copied per tick before yielding. - #[serde(default = "default_batch_chunks")] - pub batch_chunks: usize, - - /// Where the volume lock lives, overriding the filesystem this node's root sits on. - /// - /// `None` in production, which keys the lock by device id so every node on one disk - /// serialises against the others. Tests set it so a test's migration contends only - /// with its own, rather than with every other test sharing the machine's filesystem. - /// - /// Never serialised: it exists to scope a test, not to configure a node. - #[serde(skip)] - pub lock_dir: Option, -} - -const fn default_true() -> bool { - true -} - -/// Whether this build deletes the legacy environment once the gate is satisfied. -/// -/// **`true` in this release.** Deleting `chunks.mdb` is the only step that returns disk, -/// and a build that ships with it off is a migration that never finishes: the fleet -/// already deleted 2.29M chunks out of LMDB and got back nothing, because LMDB does not -/// return freed pages to the filesystem. Every gate in front of this is still enforced, -/// and `ANT_MIGRATION_RETIRE_LEGACY=0` turns it off on a single node if one is ever -/// needed to hold both stores. -pub const RELEASE_RETIRE_LEGACY: bool = true; - -/// Environment override for the directory the per-volume migration lock lives in. -/// -/// Set this where the default cannot work, and the default cannot work wherever the nodes -/// sharing a disk do not share a `/tmp`. Our own multi-node hosts are exactly that case: -/// the systemd unit sets `PrivateTmp=true`, which gives every unit a tmpfs of its own, so -/// each node creates the same lock filename in a different filesystem, every one of them -/// takes it, and the lock serialises nothing. Point every node on a host at one directory -/// they can all write and the lock does what it is for. -/// -/// The directory must be writable by the node. A path that cannot be used is reported and -/// the node migrates unserialised, which is the same answer as having no lock, so a typo -/// here is loud rather than silent. -pub const LOCK_DIR_ENV: &str = "ANT_MIGRATION_LOCK_DIR"; - -/// Environment override for [`RELEASE_RETIRE_LEGACY`], for a canary node. -pub const RETIRE_LEGACY_ENV: &str = "ANT_MIGRATION_RETIRE_LEGACY"; - -/// Read a boolean override from the environment, falling back to the build constant. -fn env_override(name: &str, build_default: bool) -> bool { - let Ok(raw) = std::env::var(name) else { - return build_default; - }; - match raw.trim().to_ascii_lowercase().as_str() { - "1" | "true" | "yes" | "on" => true, - "0" | "false" | "no" | "off" => false, - other => { - warn!("{name}={other} is not a boolean; using the build default {build_default}"); - build_default - } - } -} - -/// The retirement switch for this build, after any environment override. -fn release_retire_legacy() -> bool { - env_override(RETIRE_LEGACY_ENV, RELEASE_RETIRE_LEGACY) -} - -const fn default_shed_hold_hours() -> u64 { - 72 -} - -const fn default_retire_delay_hours() -> u64 { - MIN_RETIRE_DELAY_HOURS -} - -const fn default_wave_hours() -> u64 { - 24 -} - -const fn default_copier_slack_mb() -> u64 { - 2048 -} - -const fn default_copier_throttle_mib_per_sec() -> u64 { - 32 -} - -const fn default_tick_secs() -> u64 { - 30 -} - -const fn default_batch_chunks() -> usize { - 64 -} - -impl Default for MigrationConfig { - fn default() -> Self { - Self { - enabled: true, - dual_write_legacy: true, - allow_shed: true, - retire_legacy: release_retire_legacy(), - shed_hold_hours: default_shed_hold_hours(), - retire_delay_hours: default_retire_delay_hours(), - wave_hours: default_wave_hours(), - copier_slack_mb: default_copier_slack_mb(), - copier_throttle_mib_per_sec: default_copier_throttle_mib_per_sec(), - tick_secs: default_tick_secs(), - batch_chunks: default_batch_chunks(), - lock_dir: None, - } - } -} - -impl MigrationConfig { - /// The retire delay, never shorter than the retention contract allows. - #[must_use] - pub fn effective_retire_delay_hours(&self) -> u64 { - self.retire_delay_hours.max(MIN_RETIRE_DELAY_HOURS) - } - - /// Copier slack in bytes. - #[must_use] - pub fn copier_slack_bytes(&self) -> u64 { - self.copier_slack_mb.saturating_mul(1024 * 1024) - } -} - -/// Where a node is in the migration. -/// -/// The phase is persisted, but only as a *decision* record. Everything derivable from -/// the filesystem is re-derived at every start: which keys are still legacy-only is -/// simply "in the LMDB and not in the file store", so an interrupted copy resumes for -/// free with no progress bookkeeping to corrupt. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum MigrationPhase { - /// Copying the legacy environment into files. Reads are the union of both stores, - /// writes go to files, and the commitment still covers everything. - Bridging, - /// The node has settled on what it will keep and commits only to its file-backed - /// keys. It keeps serving the rest from LMDB until they stop being answerable. - Committed, - /// No legacy environment. Steady state, and where every fresh node starts. - FilesOnly, -} - -/// The persisted migration marker. -/// -/// Two facts genuinely need to survive a restart: when this build first ran (so the shed -/// hold is not restarted by a reboot loop) and when the node committed to its file-backed -/// set (so the retirement clock is not either). Everything else is re-derived. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct MigrationState { - /// Marker schema version. - pub schema: u32, - /// Where the node is. - pub phase: MigrationPhase, - /// Unix seconds when this build first started on this node. - pub first_start_unix: u64, - /// Unix seconds when the node committed to its file-backed key set. - pub committed_at_unix: Option, - /// Commitment rebuilds observed since committing. - pub rebuilds_since_commit: u32, - /// How many keys the node decided not to keep, for the operator's benefit. - pub shed_key_count: u64, - /// How many chunks the file store held when the node committed. - /// - /// Cross-checked at open. A marker claiming the node is past the copying stage while - /// the file store is far emptier than it said means the two disagree about reality, - /// and the filesystem wins. - #[serde(default)] - pub kept_key_count: u64, -} - -impl MigrationState { - /// A fresh marker for a node that has just started. - #[must_use] - pub fn new(phase: MigrationPhase) -> Self { - Self { - schema: STATE_SCHEMA, - phase, - first_start_unix: now_unix(), - committed_at_unix: None, - rebuilds_since_commit: 0, - shed_key_count: 0, - kept_key_count: 0, - } - } - - /// Load the marker, writing a fresh one if there is none. - /// - /// Persisting immediately matters: `first_start_unix` is what the shed hold counts - /// from, and a marker that is only written at the first phase change would reset that - /// clock on every restart before then, so a node that restarts more often than the - /// hold would never become eligible to shed and never finish migrating. - pub fn load_or_create(root_dir: &Path, phase: MigrationPhase) -> Self { - let state = Self::load_or_new(root_dir, phase); - // Written whenever the disk does not already hold what this process is going to - // use, rather than only when there is no file at all. A marker that is present but - // could not be used is replaced in memory and was previously left on disk, so the - // next start read the same bad file and stamped `first_start_unix` afresh. That is - // precisely the reset this function exists to prevent, and it is worse than the - // one it does prevent: it repeats. Three ways in, all of them leaving a file that - // exists: a truncated or otherwise unparseable marker, one from a newer schema, and - // one whose clock is impossible and is corrected by `with_sane_clocks`. - // - // A node restarting more often than the shed hold then never becomes eligible to - // shed and never finishes migrating, and a node short of disk is exactly the node - // that restarts. - let raw = std::fs::read(state_path(root_dir)).ok(); - let on_disk = raw - .as_deref() - .and_then(|bytes| serde_json::from_slice::(bytes).ok()); - if on_disk.as_ref() == Some(&state) { - return state; - } - // The schema is read on its own, from the raw JSON, rather than taken from a - // successful parse into today's struct. A marker from a genuinely newer build is - // exactly the one least likely to parse into it: a phase this build has no name - // for, a field that changed type, a field that went away. Reading the schema only - // when the whole thing parses means the markers most worth keeping are the ones - // that would be written over. - let newer_schema = raw - .as_deref() - .and_then(|bytes| serde_json::from_slice::(bytes).ok()) - .and_then(|value| value.get("schema").and_then(serde_json::Value::as_u64)) - .filter(|schema| *schema > u64::from(STATE_SCHEMA)); - if let Some(schema) = newer_schema { - let Some(kept) = free_kept_marker_path(root_dir, schema) else { - warn!( - "A newer migration marker (schema {schema}) is here and every name to \ - keep it under is taken. Leaving it, which means the shed hold restarts \ - on every boot until it is dealt with" - ); - return state; - }; - if let Err(e) = std::fs::rename(state_path(root_dir), &kept) { - warn!( - "Could not move the newer migration marker aside ({e}); leaving it, \ - which means the shed hold restarts on every boot until it is dealt with" - ); - return state; - } - info!( - "Kept the newer migration marker as {} and started one this build can use", - kept.display() - ); - } - if let Err(e) = state.save(root_dir) { - warn!("Could not write the migration marker: {e}"); - } - state - } - - /// Load the marker, or start a fresh one. - /// - /// An unreadable marker is replaced rather than fatal: it is a hint, and every fact - /// it holds is either recoverable or conservative to reset. Losing it restarts the - /// shed hold and the retirement clock, which delays a migration and never rushes one. - pub fn load_or_new(root_dir: &Path, phase: MigrationPhase) -> Self { - let path = state_path(root_dir); - let Ok(bytes) = std::fs::read(&path) else { - return Self::new(phase); - }; - match serde_json::from_slice::(&bytes) { - Ok(state) if state.schema <= STATE_SCHEMA => state.with_sane_clocks(), - Ok(state) => { - warn!( - "Migration marker {} was written by a newer build (schema {}); \ - starting a fresh one", - path.display(), - state.schema - ); - Self::new(phase) - } - Err(e) => { - warn!( - "Migration marker {} is unreadable ({e}); starting a fresh one. \ - The shed hold and retirement clock restart from now.", - path.display() - ); - Self::new(phase) - } - } - } - - /// Persist the marker so a reader sees either the old content or the new. - /// - /// # Errors - /// - /// Returns [`Error::Storage`] if the marker cannot be written. - pub fn save(&self, root_dir: &Path) -> Result<()> { - let path = state_path(root_dir); - let bytes = serde_json::to_vec_pretty(self) - .map_err(|e| Error::Storage(format!("Failed to encode migration marker: {e}")))?; - crate::storage::file_store::write_file_durably(&path, &bytes)?; - debug!("Migration marker updated: phase {:?}", self.phase); - Ok(()) - } - - /// Replace timestamps that cannot be true with "now". - /// - /// Zero and future values both make a hold vacuous, and a node whose clock was not - /// yet synchronised at first boot writes zero without anyone tampering. Resetting to - /// now delays a migration, which is the safe direction. - #[must_use] - fn with_sane_clocks(mut self) -> Self { - let now = now_unix(); - if self.first_start_unix == 0 || self.first_start_unix > now { - warn!("Migration marker has an implausible first-start time; restarting the hold"); - self.first_start_unix = now; - } - self.committed_at_unix = self.committed_at_unix.map(|at| { - if at == 0 || at > now { - warn!("Migration marker has an implausible commit time; restarting the clock"); - now - } else { - at - } - }); - self - } - - /// Whether the shed hold has elapsed. - #[must_use] - pub fn shed_hold_elapsed(&self, config: &MigrationConfig) -> bool { - let hold = config.shed_hold_hours.saturating_mul(3600); - now_unix().saturating_sub(self.first_start_unix) >= hold - } - - /// Whether the retirement delay has elapsed since committing. - #[must_use] - pub fn retire_delay_elapsed(&self, config: &MigrationConfig) -> bool { - let Some(at) = self.committed_at_unix else { - return false; - }; - let delay = config.effective_retire_delay_hours().saturating_mul(3600); - now_unix().saturating_sub(at) >= delay - } -} - -/// A name to keep a newer marker under that nothing is using yet. -/// -/// The plain `.schema-N` name is deterministic, so a node downgraded twice would otherwise -/// write over the marker it kept the first time, or fail the rename and restart the hold on -/// every boot. Returns `None` if every name is taken, which the caller reports rather than -/// destroying anything. -fn free_kept_marker_path(root_dir: &Path, schema: u64) -> Option { - for attempt in 0..16u32 { - // Appended, not `with_extension`, which would replace the `.json` and leave a name - // that no longer says what the file is. - let mut candidate = state_path(root_dir).into_os_string(); - candidate.push(format!(".schema-{schema}")); - if attempt > 0 { - candidate.push(format!(".{attempt}")); - } - let candidate = PathBuf::from(candidate); - // `symlink_metadata`, not `try_exists`. The latter follows links, so a dangling - // symbolic link at this name reads as nothing being there while `rename` would - // happily replace it. Anything at all, of any kind, means pick another name. - if std::fs::symlink_metadata(&candidate).is_err() { - return Some(candidate); - } - } - None -} - -/// Path of the persisted marker. -#[must_use] -pub fn state_path(root_dir: &Path) -> PathBuf { - root_dir.join(MIGRATION_STATE_FILE) -} - -/// Seconds since the Unix epoch, saturating at zero if the clock is before it. -#[must_use] -pub fn now_unix() -> u64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .map_or(0, |d| d.as_secs()) -} - -/// A host-wide advisory lock that serialises migrations sharing one volume. -/// -/// Twelve nodes on one 492 GiB volume each need roughly their live payload free to copy -/// and each return rather more when they retire, so one at a time the host gains space -/// and the queue accelerates. All twelve at once need twelve times the space and all -/// twelve stall. The lock is taken non-blocking: a node that cannot get it simply waits -/// for the next tick. -/// -/// Where the lock file sits is `lock_path_for`'s decision (private, so this is not a -/// link), and this doc used to describe -/// a branch of it that a running node almost never reaches: the parent of the node root is -/// the last resort, taken only when the root's own metadata cannot be read. What a node -/// normally uses is the host's temporary directory keyed by the volume's device id, or the -/// directory named by [`LOCK_DIR_ENV`] when one is set. -/// -/// Which of those is right is a fact about the deployment that no node can check for -/// itself, and getting it wrong is silent: every node takes a lock of its own and reports -/// success. That is why the path is logged when the lock is taken, and why a host whose -/// nodes do not share a `/tmp` has to be told where the lock lives. -/// -/// The directory has to be one only the node's own user can write. A predictable path in a -/// world-writable `/tmp` can be created and held by any local user, who could then keep -/// every node on the host from ever migrating. -#[derive(Debug)] -pub struct VolumeLock { - /// The held file. Dropping it releases the lock. - file: std::fs::File, - /// Where it lives, for logging. - #[cfg_attr(not(feature = "logging"), allow(dead_code))] - path: PathBuf, -} - -/// The result of asking for the volume lock. -pub enum LockAttempt { - /// This node has it. - Acquired(VolumeLock), - /// Another node on the volume is migrating. Wait. - Busy, - /// No lock is possible here at all, so proceed unserialised. - /// - /// Kept distinct from `Busy` because conflating the two silently strands any node - /// whose parent directory is not writable: it would wait forever for a lock nobody - /// holds. - Unavailable, -} - -impl VolumeLock { - /// Try to take the lock for the volume hosting `root_dir`. - #[must_use] - pub fn try_acquire(root_dir: &Path, scope: Option<&Path>) -> LockAttempt { - use fs2::FileExt; - let path = scope.map_or_else( - || lock_path_for(root_dir), - |dir| dir.join("ant-migration.lock"), - ); - let file = match std::fs::OpenOptions::new() - .write(true) - .create(true) - .truncate(false) - .open(&path) - { - Ok(f) => f, - Err(e) => { - warn!( - "Could not create the migration lock at {}: {e}. This node will \ - migrate without serialising against others on the same volume, so \ - watch its free space.", - path.display() - ); - return LockAttempt::Unavailable; - } - }; - match file.try_lock_exclusive() { - Ok(()) => { - // Info, not debug. Whether the lock is doing anything depends on whether - // the neighbours on this disk can see the same path, which is a deployment - // fact no node can check. Printing the path is what lets somebody answer it - // from a log rather than by reading a unit file. - info!("Took the volume migration lock at {}", path.display()); - LockAttempt::Acquired(Self { file, path }) - } - // Only contention means another node is migrating. Everything else, a - // filesystem that does not implement locking at all being the one that - // matters, is a lock this node will never get, and reporting it as contention - // would leave it waiting forever for a holder that does not exist. - Err(e) if is_lock_contention(&e) => LockAttempt::Busy, - Err(e) => { - warn!( - "Could not lock {}: {e}. This node will migrate without serialising \ - against others on the same volume, so watch its free space.", - path.display() - ); - LockAttempt::Unavailable - } - } - } -} - -/// Is this the error a lock held by someone else produces? -fn is_lock_contention(e: &std::io::Error) -> bool { - e.kind() == std::io::ErrorKind::WouldBlock - || (e.raw_os_error().is_some() - && e.raw_os_error() == fs2::lock_contended_error().raw_os_error()) -} - -/// Where the lock for the volume hosting `root_dir` lives. -/// -/// Keyed by the filesystem, not by the path. Two nodes on one host are configured with -/// different roots by definition, so a lock beside the root serialises a node against -/// nobody: `/srv/node-a/data` and `/srv/node-b/data` would take two different locks on one -/// disk and copy at the same time, which is the case the lock exists to prevent. -/// -/// The device id names the filesystem, and the host's temporary directory is somewhere -/// every node on that host can reach. If the device cannot be read, this falls back to a -/// lock beside the root: weaker, but never worse than having none. -/// -/// **That last sentence is only true where the nodes share a `/tmp`.** Where they do not, -/// each computes the same name in a filesystem of its own, every one of them takes it, and -/// the lock serialises nothing while logging that it worked. `PrivateTmp=true` in a systemd -/// unit does exactly that, and our own worker unit sets it. There is no way to tell from -/// inside one process whether the `/tmp` it can see is the one its neighbours see, so this -/// cannot be detected here and has to be configured: [`LOCK_DIR_ENV`] names a directory -/// every node on the host can reach, and is consulted first. -fn lock_path_for(root_dir: &Path) -> PathBuf { - lock_path_with(root_dir, std::env::var(LOCK_DIR_ENV).ok().as_deref()) -} - -/// The same decision, with the configured directory passed in rather than read. -/// -/// Split so it can be tested without touching process-wide environment. A test that set -/// `TMPDIR` to stage the private-`/tmp` case would change where every other test's -/// `TempDir` lands, and then delete it underneath them: run in parallel that takes out -/// dozens of unrelated tests with LMDB failures that look like anything but their cause. -fn lock_path_with(root_dir: &Path, configured: Option<&str>) -> PathBuf { - // Before anything derived, because an operator who has set this knows something about - // the host that this function cannot find out. - if let Some(dir) = configured { - let dir = dir.trim(); - if !dir.is_empty() { - return Path::new(dir).join("ant-migration.lock"); - } - } - #[cfg(unix)] - { - use std::os::unix::fs::MetadataExt; - if let Ok(meta) = std::fs::metadata(root_dir) { - return std::env::temp_dir().join(format!("ant-migration-{}.lock", meta.dev())); - } - } - // Resolved first, because a relative root has no volume in it to read. Two nodes - // started from different working directories on one drive would otherwise each fall - // through to a lock beside their own root, which serialises neither against the other. - #[cfg(not(unix))] - let resolved = std::fs::canonicalize(root_dir).unwrap_or_else(|_| root_dir.to_path_buf()); - #[cfg(not(unix))] - let root_dir = resolved.as_path(); - // Off Unix, the volume root: the drive or share the path starts from. Not as precise - // as a device id, since a mount point below it belongs to another volume, but it - // groups the ordinary case of several nodes under one drive letter, which is what a - // lock beside each node's own root does not. - #[cfg(not(unix))] - { - use std::path::Component; - if let Some(Component::Prefix(prefix)) = root_dir.components().next() { - let key: String = prefix - .as_os_str() - .to_string_lossy() - .chars() - .filter(char::is_ascii_alphanumeric) - .collect(); - if !key.is_empty() { - return std::env::temp_dir().join(format!("ant-migration-{key}.lock")); - } - } - } - root_dir - .parent() - .unwrap_or(root_dir) - .join("ant-migration.lock") -} - -impl Drop for VolumeLock { - fn drop(&mut self) { - use fs2::FileExt; - if let Err(e) = FileExt::unlock(&self.file) { - debug!( - "Releasing the migration lock {} failed: {e}", - self.path.display() - ); - } - } -} - -/// Summary of what the copier moved during one pass. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -pub struct CopyReport { - /// Chunks copied into the file store. - pub copied: u64, - /// Bytes copied. - pub bytes: u64, - /// Keys skipped because the legacy bytes did not hash to their address, or were - /// larger than a chunk may be. - pub unusable: u64, - /// Keys that could not be copied for a reason that may clear on a later pass. - pub failed: u64, - /// Keys that had vanished from the legacy store between the scan and the copy. - pub vanished: u64, - /// Whether the pass stopped because free space reached the slack floor. - pub stopped_for_space: bool, -} - -impl CopyReport { - /// Fold another pass into this one. - pub fn merge(&mut self, other: Self) { - self.copied += other.copied; - self.bytes += other.bytes; - self.unusable += other.unusable; - self.failed += other.failed; - self.vanished += other.vanished; - self.stopped_for_space |= other.stopped_for_space; - } -} - -/// How many waves a close group is divided into, so that at most -/// [`CONCURRENT_MIGRATIONS_PER_GROUP`] of it are giving chunks up at once. -#[must_use] -pub fn migration_wave_count(close_group_size: usize) -> u64 { - let group = close_group_size.max(1) as u64; - let per_wave = CONCURRENT_MIGRATIONS_PER_GROUP.max(1) as u64; - group.div_ceil(per_wave).max(1) -} - -/// Which wave this node belongs to, derived from its own ID. -/// -/// Deterministic and needs no coordination, which is the point: a node cannot ask its -/// close group "are you migrating?" without a protocol change, and the answer would be -/// stale by the time it arrived. Hashing the peer ID spreads the members of any group -/// across the waves without anybody agreeing on anything. -/// -/// It is a stagger, not a guarantee. Seven IDs hashed into four waves will not always land -/// two, two, two, one. What makes it safe rather than merely tidy is that it composes with -/// the possession gate: a node whose turn has come still cannot give a chunk up until its -/// neighbours have proven they hold it, so an unlucky wave waits instead of over-shedding. -#[must_use] -pub fn migration_wave_for(self_id: Option<&PeerId>, close_group_size: usize) -> u64 { - let waves = migration_wave_count(close_group_size); - let Some(peer) = self_id else { - return 0; - }; - let digest = blake3::hash(&[MIGRATION_WAVE_DOMAIN, peer.as_bytes().as_slice()].concat()); - let mut head = [0u8; 8]; - head.copy_from_slice(digest.as_bytes().get(..8).unwrap_or(&[0u8; 8])); - u64::from_le_bytes(head) % waves -} - -/// Domain separator so the wave assignment cannot be confused with any other use of a -/// hashed peer ID. -const MIGRATION_WAVE_DOMAIN: &[u8] = b"ant-node/storage-migration-wave/v1"; - -/// Whether this node's wave has opened yet. -/// -/// Wave `w` opens `w * wave_hours` after this build first started on this node. A node -/// that has room for everything never consults this: it copies and retires without ever -/// being unable to serve, so it is not part of the problem the waves exist to solve. -#[must_use] -pub fn wave_has_opened(state: &MigrationState, config: &MigrationConfig, wave: u64) -> bool { - now_unix() >= wave_opens_at(state, config, wave) -} - -/// When a given wave opens, in Unix seconds. -/// -/// Measured from the END of the shed hold, not from first start. Measured from the start -/// the two settings cancel each other out: with a 72 hour hold and 24 hour waves, waves -/// would open at 0, 24, 48 and 72 hours while nothing at all may shed until hour 72, so -/// every wave would be open the moment the first one could act and the whole close group -/// would migrate together. That is the pile-up the waves exist to prevent. -#[must_use] -pub fn wave_opens_at(state: &MigrationState, config: &MigrationConfig, wave: u64) -> u64 { - state - .first_start_unix - .saturating_add(config.shed_hold_hours.saturating_mul(3600)) - .saturating_add(wave.saturating_mul(config.wave_hours.saturating_mul(3600))) -} - -/// Order keys closest-first by XOR distance from this node. -/// -/// Shedding walks this list from the far end, so the keys a node gives up are the ones it -/// is furthest from, and therefore the ones its close group covers best. -#[must_use] -pub fn rank_closest_first(mut keys: Vec, self_xor: Option) -> Vec { - let Some(me) = self_xor else { - // No identity available (devnet, unit tests). Ascending key order is stable and - // deterministic, which is all the copier needs. - keys.sort_unstable(); - return keys; - }; - keys.sort_unstable_by_key(|k| crate::client::xor_distance(k, &me)); - keys -} - -/// Structured field marking every line the fleet gate for R3 is read from. -/// -/// R3 ships when the fleet shows migrations have finished, so these lines have to be -/// queryable rather than merely readable. One field name, three values. -pub const MIGRATION_EVENT: &str = "migration_event"; - -/// Log the operator-facing summary of a completed migration. -/// -/// `freed_bytes` is what the retired environment held, which is what the deletion running -/// in the background will return. The line that says the space is actually back is -/// `migration_event = "space_returned"`, emitted by that deletion when it finishes. Two -/// lines rather than one because the deletion of a large environment takes minutes, and a -/// node that reports the disk back before it is back is a node whose operator cannot tell -/// a slow deletion from a failed one. -pub fn log_migration_complete(kept: u64, shed: u64, freed_bytes: u64) { - #[allow(clippy::cast_precision_loss)] // display only - let freed_gib = freed_bytes as f64 / (1024.0 * 1024.0 * 1024.0); - if shed == 0 { - info!( - migration_event = "complete", - kept, - shed, - freed_bytes, - "Storage migration complete: {kept} chunks now in the file store, nothing shed, \ - {freed_gib:.2} GiB being returned to the filesystem" - ); - } else { - info!( - migration_event = "complete", - kept, - shed, - freed_bytes, - "Storage migration complete: kept {kept} chunks, shed {shed} that would not fit, \ - {freed_gib:.2} GiB being returned to the filesystem. The shed keys are the ones this \ - node was furthest from; replication will refetch what still belongs here now \ - that there is room." - ); - } -} - -// ──────────────────────────────────────────────────────────────────────────── -// The driver -// ──────────────────────────────────────────────────────────────────────────── - -/// How many positions from the end of the admission group a node may give up. -/// -/// A chunk has exactly one holder at each rank, so restricting shedding to the last two -/// positions means only two of its holders ever consider dropping it, and the staged -/// rollout brings that to one. Without this rule the property is only statistical: -/// every holder could be short of space at once, each shed the same chunk, and the -/// per-volume lock would not know, because it serialises one volume and this is a -/// network-wide question. -/// -/// Measured against [`storage_admission_width`], not the close group, so the migration is -/// never more willing to drop a chunk than the pruner is. The pruner treats the wider -/// group as strictly in-range and refuses to delete inside it; shedding ranks that the -/// pruner protects would make a one-off migration weaker than the thing that runs every -/// day. -pub const SHEDDABLE_TAIL_RANKS: usize = 2; - -/// How many nodes of one close group may be giving chunks up at the same time. -/// -/// The close group is the unit that matters, not the volume and not the fleet. If every -/// holder of a chunk migrates at once, none of them can prove to the others that a copy -/// survives, and the whole group deadlocks waiting on each other. Holding it to two means -/// the other five are steady, can answer possession challenges, and are still serving the -/// chunk while the two rebuild. -pub const CONCURRENT_MIGRATIONS_PER_GROUP: usize = 2; - -/// How many keys a refusal names, so the log stays readable. -const REFUSAL_SAMPLE: usize = 4; - -/// How recently a peer must have published a commitment to be trusted as a holder. -/// -/// Commitments rotate hourly and are gossiped on the neighbour-sync cadence, so a peer -/// that has not published one for this long is not simply quiet: it has either stopped -/// speaking the protocol or retired its commitment and not yet rotated a new one. The -/// second is exactly what a node in the middle of its own migration looks like, and -/// counting it as a holder is how two migrating nodes could each conclude the other was -/// covering the chunk. -const COMMITMENT_FRESHNESS: Duration = Duration::from_secs(2 * 3600); - -/// How many keys one possession round asks about. -/// -/// The round batches by peer, so this bounds the size of a single request rather than the -/// number of requests. -const POSSESSION_BATCH_KEYS: usize = 256; - -/// How long to wait before re-evaluating a shed decision that was refused. -const SHED_REEVALUATION_INTERVAL: Duration = Duration::from_secs(600); - -/// How many copied chunks between operator-facing progress lines. -const PROGRESS_LOG_EVERY: usize = 500; - -/// How long a clean pre-retirement verification stays usable. -/// -/// Chunks written since the pass were content-checked on the way in and flushed, so the -/// only thing the window exposes is bit rot in the last half hour, which is the ordinary -/// risk of any file and is caught on read. -const VERIFICATION_REUSE_WINDOW: Duration = Duration::from_secs(1800); - -/// The network facts the driver needs, kept behind one type so the store itself stays -/// free of any knowledge of routing or commitments. -pub struct MigrationContext { - /// Routing, for close-group rank and possession checks. `None` in devnet and tests. - pub p2p: Option>, - /// This node's peer ID. - pub self_id: Option, - /// This node's address in the key space, for ordering the copy closest-first. - pub self_xor: Option, - /// The responder commitment state, which owns the retention contract. - pub commitment: Option>, - /// Replication settings, for the possession round that gates shedding. - pub replication: Option>, - /// Neighbour-sync state, which the possession challenge needs. - pub sync_state: Option>>, - /// Coordinator for the possession challenges. - pub audit_challenge_coordinator: - Option>, - /// What this node last heard each peer commit to. - /// - /// Used to require that a peer trusted to hold a chunk is currently publishing a - /// claim, rather than sitting between a retired commitment and its next rotation, - /// which is precisely the state a node in the middle of its own migration is in. - pub peer_commitments: Option< - Arc< - tokio::sync::RwLock< - HashMap, - >, - >, - >, - /// Close-group width. - pub close_group_size: usize, -} - -/// How many of the peers auditing this node now have seen its reduced commitment. -/// -/// `received` is who was sent the current root, `current` is the close group as routing -/// sees it at this moment. Only the overlap counts. A peer that received the root and has -/// since left is not going to audit this node, and a peer that has since joined has never -/// seen the root, so neither is evidence that shedding is safe. -fn enough_of_the_group_knows( - received: &HashSet, - current: &[PeerId], - needed: usize, -) -> bool { - if needed == 0 { - return false; - } - current.iter().filter(|p| received.contains(*p)).count() >= needed -} - -impl MigrationContext { - /// How many peers of the close group must have seen the reduced commitment. - /// - /// The same tolerance the pruner applies to possession proofs: all of them for a group - /// of one or two, one short of the group otherwise, so a single unreachable peer - /// cannot veto the migration forever without accepting an uninformed close group. - #[must_use] - pub fn commitment_recipients_needed(&self) -> usize { - prune_proofs_needed(self.close_group_size.saturating_sub(1)) - } - - /// Have enough of this node's close group actually received its reduced commitment? - /// - /// A rotation is not the same as neighbours knowing. Until they have seen the smaller - /// key set they keep auditing against the one this node used to hold, so giving a - /// chunk up before then turns a legitimate migration into a wave of audit failures. - pub async fn neighbours_know_the_commitment(&self) -> bool { - let needed = self.commitment_recipients_needed(); - if needed == 0 { - return false; - } - let Some(state) = self.commitment.as_ref() else { - return false; - }; - let received = state.current_delivered_peers(); - if received.is_empty() { - return false; - } - // Counted against the group as it stands now, not as it stood when the root went - // out. A peer that has since left knowing this node's reduced commitment says - // nothing about the peers that will actually audit it, and letting a departed - // peer satisfy the gate is how a node gives chunks up while its real neighbours - // still hold it to the larger key set. - let Some(current) = self.current_close_group().await else { - return false; - }; - enough_of_the_group_knows(&received, ¤t, needed) - } - - /// This node's close group as routing sees it now, or `None` if the view is too thin - /// to be evidence about a group at all. - async fn current_close_group(&self) -> Option> { - let (Some(p2p), Some(me), Some(self_xor)) = ( - self.p2p.as_ref(), - self.self_id.as_ref(), - self.self_xor.as_ref(), - ) else { - return None; - }; - // Self-inclusive, then self filtered out. The self-excluding call would return - // `close_group_size` *remote* peers, one more than the group actually has, and the - // threshold is computed from a group that includes this node. Four real - // neighbours plus one peer outside the group would then clear a bar meant to - // require five real ones. - let closest = p2p - .dht_manager() - .find_closest_nodes_local_with_self(self_xor, self.close_group_size) - .await; - let peers: Vec = closest - .iter() - .map(|n| n.peer_id) - .filter(|p| p != me) - .collect(); - if peers.len() + 1 < self.close_group_size { - return None; - } - Some(peers) - } - - /// Is this key still answerable under a retained commitment slot? - /// - /// This is the pruner's existing veto, reused verbatim: a key the node could still - /// be challenged on must not lose its last local copy. - #[must_use] - pub fn still_answerable(&self, key: &XorName) -> bool { - self.commitment - .as_ref() - .is_some_and(|state| state.is_held(key)) - } - - /// The width this node measures ranks against: the admission group, not the close - /// group. - #[must_use] - pub fn shed_width(&self) -> usize { - storage_admission_width(self.close_group_size) - } - - /// This node's position in `key`'s admission group. - pub async fn close_group_rank(&self, key: &XorName) -> GroupRank { - let (Some(p2p), Some(me)) = (self.p2p.as_ref(), self.self_id.as_ref()) else { - return GroupRank::Unknown; - }; - let closest = p2p - .dht_manager() - .find_closest_nodes_local_with_self(key, self.shed_width()) - .await; - closest - .iter() - .position(|n| n.peer_id == *me) - .map_or(GroupRank::Outside, GroupRank::Inside) - } - - /// May this node give up `key` without risking its last replica? - /// - /// Only if it is outside the admission group entirely, or sits in that group's last - /// [`SHEDDABLE_TAIL_RANKS`] positions. Never when the answer is unknown. - pub async fn may_shed(&self, key: &XorName) -> bool { - rank_is_sheddable(self.close_group_rank(key).await, self.shed_width()) - } -} - -/// Where this node sits in a key's admission group. -/// -/// `Unknown` is deliberately distinct from `Outside`. Collapsing the two would turn "this -/// node has no routing table to consult" into "no other node is closer", which is a -/// licence to give up every chunk on no evidence whatsoever. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum GroupRank { - /// This node is at this position, counting from the closest. - Inside(usize), - /// This node is not among the closest for this key. - Outside, - /// Routing state is unavailable, so the question cannot be answered. - Unknown, -} - -/// Which of `keys` this node has no proof anyone else is holding. -/// -/// This is the gate on giving a chunk up at all, and it is deliberately the **same** -/// evidence the pruner demands before it deletes: cryptographic possession proofs from -/// all but one of the key's current close group, which is six of seven at -/// production width. -/// -/// Rank alone was not enough. Being far from a chunk says something about who *should* -/// hold it, not about who *does*, and during a fleet-wide migration the nodes that should -/// hold it are exactly the ones that may also be short of space. Nor is the cheap -/// `VerificationRequest` enough: it carries a self-reported `present: bool`, and a node -/// that has silently lost a chunk still answers yes. The challenge here makes a peer -/// return a digest over a nonce it has never seen, which it cannot do without the bytes. -/// -/// Returns the keys that failed, so the caller can name them. An empty result means every -/// key asked about is proven to live somewhere else. -/// -/// Without routing state, every key is unconfirmed: no view of the network is no evidence. -pub async fn unconfirmed_by_neighbours( - store: &Arc, - context: &MigrationContext, - keys: &[XorName], -) -> Vec { - let (Some(p2p), Some(self_id), Some(config), Some(sync_state), Some(coordinator)) = ( - context.p2p.as_ref(), - context.self_id.as_ref(), - context.replication.as_ref(), - context.sync_state.as_ref(), - context.audit_challenge_coordinator.as_ref(), - ) else { - return keys.to_vec(); - }; - - let local_key_count = - usize::try_from(store.current_chunks().unwrap_or(0)).unwrap_or(usize::MAX); - let dht = p2p.dht_manager(); - let mut unconfirmed = Vec::new(); - - for batch in keys.chunks(POSSESSION_BATCH_KEYS) { - // Ask only the peers that are currently closest to each key. A proof from a peer - // that has since moved out of the group is not evidence the chunk will stay there. - let mut targets_by_key: HashMap> = HashMap::new(); - let mut keys_by_peer: HashMap> = HashMap::new(); - for key in batch { - // Self-inclusive, matching the pruner, whose evidence this is. The - // self-excluding call returns one peer more than the key's group holds, and a - // proof from a peer outside it is not evidence the chunk stays in it. - let closest = dht - .find_closest_nodes_local_with_self(key, config.close_group_size) - .await; - let peers: Vec = closest - .iter() - .map(|n| n.peer_id) - .filter(|p| p != self_id) - .collect(); - for peer in &peers { - keys_by_peer.entry(*peer).or_default().push(*key); - } - targets_by_key.insert(*key, peers); - } - - let proofs = prove_peers_hold_records( - &keys_by_peer, - local_key_count, - store, - p2p, - config, - sync_state, - coordinator, - ) - .await; - - // A proof is necessary but not sufficient. The peer must also be currently - // publishing a commitment, so a node that has retired its own and not yet rotated - // a replacement, which is what a node mid-migration looks like, is not counted as - // the reason this node may give a chunk up. - let publishing = peers_publishing_a_recent_commitment(context).await; - - for key in batch { - let group = targets_by_key.get(key).map_or(&[][..], Vec::as_slice); - - // The lookup is self-inclusive, and this node is giving the chunk up, so a - // full group is `close_group_size` peers none of which is this node. Fewer - // than that is a routing view too thin to be evidence about the group at all, - // which is what a table looks like shortly after a restart. This node still - // appearing in the group means it is not outside it after all, and the - // decision to give the chunk up was taken against a view that has since - // changed. - if group.len() < config.close_group_size { - unconfirmed.push(*key); - continue; - } - - // The threshold comes from the configured group size, never from whichever - // subset happens to qualify, nor from however many peers routing returned. - // Deriving it from the filtered list is how two last holders destroy a chunk - // between them: each sees only the other publishing, so each needs exactly one - // proof, each gets it from the other, and both delete. Deriving it from the - // observed length is the same mistake more quietly: a view that has lost a - // peer lowers the bar exactly when it should not be trusted. - let needed = prune_proofs_needed(config.close_group_size); - let qualifying: Vec = group - .iter() - .filter(|p| publishing.contains(*p)) - .copied() - .collect(); - if !target_peers_reported_present(key, &qualifying, &proofs, needed) { - unconfirmed.push(*key); - } - } - } - unconfirmed -} - -/// The peers this node has heard a commitment from recently enough to trust as holders. -async fn peers_publishing_a_recent_commitment(context: &MigrationContext) -> HashSet { - let Some(records) = context.peer_commitments.as_ref() else { - return HashSet::new(); - }; - records - .read() - .await - .iter() - .filter(|(_, record)| { - record.last_commitment().is_some() - && record.received_at.elapsed() < COMMITMENT_FRESHNESS - }) - .map(|(peer, _)| *peer) - .collect() -} - -/// Whether a position in the admission group may be given up. -/// -/// Split out from the routing lookup so the rule itself is testable without a network. -/// `width` is [`storage_admission_width`], not the close-group size: a key this node is -/// outside the admission group for is one the pruner would delete anyway, and inside it -/// only the last [`SHEDDABLE_TAIL_RANKS`] positions may go. -#[must_use] -pub fn rank_is_sheddable(rank: GroupRank, width: usize) -> bool { - // A group no wider than the tail has no tail to give up. Saturating alone would set - // the threshold to zero and make every member sheddable, which is the opposite of - // what a narrow group needs. - let protected_below = if width <= SHEDDABLE_TAIL_RANKS { - width - } else { - width - SHEDDABLE_TAIL_RANKS - }; - match rank { - // No routing to ask. Never a licence: a node with no view of the network has no - // grounds at all for believing anyone else holds the chunk. - GroupRank::Unknown => false, - GroupRank::Outside => true, - GroupRank::Inside(rank) => rank >= protected_below, - } -} - -/// Whether this store needs a migration driver at all. -/// -/// The single predicate both the spawn site and its test use, so "should this node be -/// migrating" cannot be answered one way by the wiring and another way by what checks it. -#[must_use] -pub fn should_migrate(store: &Arc) -> bool { - // Or has a removal to finish, or has something at the environment's path it could not - // open. A node whose retirement was interrupted has no handle and nothing left to - // copy, but its disk has not come back. A node whose environment is a link to storage - // that was not mounted at startup has neither, and its chunks come back when the - // storage does; without a driver it would stay blind to them until a restart. - store.has_legacy() || store.has_cleanup_pending() || store.legacy_dir_is_on_disk() -} - -/// Runs the migration to completion, then returns. -/// -/// Everything it does is idempotent and derived from the filesystem, so a crash at any -/// point costs at most the work of one tick. -pub async fn run(store: Arc, context: MigrationContext, shutdown: CancellationToken) { - let config = store.migration_config().clone(); - if !worth_starting(&store, &config) { - return; - } - - let tick = Duration::from_secs(config.tick_secs.max(1)); - let mut volume_lock: Option = None; - let mut held = LockHold::default(); - let mut next_shed_evaluation = Instant::now(); - let mut next_handle_recovery = Instant::now(); - // A clean verification is a full re-read of everything both stores hold. If - // retirement is then deferred (a read still holds the legacy handle), re-hashing on - // every tick would be minutes of disk for nothing, so a recent pass is reused. - let mut verified: Option<(VerifyReport, Instant)> = None; - - loop { - tokio::select! { - () = shutdown.cancelled() => { - debug!("Storage migration stopping for shutdown"); - return; - } - () = tokio::time::sleep(tick) => {} - } - - // Never hold the volume against the rest of the machine for longer than this, - // whatever the node is waiting on. Dropping it costs a tick: if nobody else wants - // it, the branches below take it straight back. - if held.has_overstayed() { - debug!( - "Held the volume migration lock for {} hour(s); giving it back so any \ - other node on this volume gets a turn", - MAX_VOLUME_LOCK_HOLD.as_secs() / 3600 - ); - volume_lock = None; - held.give_up(); - } - - // Cleanup first. It can put an unmarked directory back under the live name, and - // recovery is what gives the node a handle to it; the other order leaves that - // until the next tick, and the completion check in between would see no handle - // and no pending cleanup and call the migration finished. - let cleanup = cleanup_state(&store); - // Before anything reads the key set: a write that nobody waited for leaves a - // note behind, and only the disk can say what became of it. - store.reconcile_pending_writes().await; - maybe_recover_lost_handle(&store, &mut next_handle_recovery).await; - if cleanup == CleanupState::Finished && !store.has_legacy() { - info!("Storage migration finished; nothing left on disk to clean up"); - return; - } - - match store.migration_phase() { - // Nothing left to migrate. Whether there is anything left to clean up is - // decided at the top of the loop, which is also where this returns from. - MigrationPhase::FilesOnly => { - volume_lock = None; - held.released(); - } - MigrationPhase::Bridging => { - // Held from the first copy through retirement, not released in between: - // a node that let go after copying would let its eleven neighbours start - // theirs before it had returned a byte, which is the exact pile-up the - // lock exists to prevent. The one exception is a node that has become - // permanently stuck (see below), which must not go on excluding the - // others for a release. - if matches!( - take_volume_lock( - &mut volume_lock, - &mut held, - store.root_dir(), - config.lock_dir.as_deref(), - ), - LockStep::WaitATick - ) { - continue; - } - if bridge_tick( - &store, - &context, - &config, - &mut next_shed_evaluation, - &shutdown, - ) - .await - { - // The copier ran. That is the volume lock being used rather than held. - held.note_disk_work(); - } else { - // Copying is blocked on something only an operator can change, so - // stop holding the volume lock against the other nodes here. - volume_lock = None; - held.released(); - } - } - MigrationPhase::Committed => { - // A node that restarted in this phase has no lock, and the work below - // (copying anything that must be kept, then re-reading the whole store to - // verify it) is exactly the disk-heavy work the lock exists to serialise. - if matches!( - take_volume_lock( - &mut volume_lock, - &mut held, - store.root_dir(), - config.lock_dir.as_deref(), - ), - LockStep::WaitATick - ) { - continue; - } - match retire_tick(&store, &context, &config, &mut verified, &shutdown).await { - // Not a return. The environment is gone from the node's point of - // view, but its directory is still being deleted in the background, - // and if that fails there has to be something left to try again. The - // loop exits at the top once nothing is pending. - RetireOutcome::Done => { - volume_lock = None; - held.released(); - } - // Time spent reading or copying is the volume lock doing its job, not - // a node sitting on it. The cap is there for a node that waits, and - // restarting a full verification because a large store took longer - // than the cap would be the cap causing the problem it prevents. - RetireOutcome::Working => held.note_disk_work(), - RetireOutcome::Waiting => {} - RetireOutcome::NoWorkToSerialise => { - // Nothing this node can do will return space, so holding the - // volume lock only stops its neighbours from trying. In R1, where - // retirement is switched off entirely, holding it would mean one - // node per volume copies and the other eleven do nothing for the - // whole release. - volume_lock = None; - held.released(); - } - } - } - } - } -} - -/// Should the driver run at all, and say why in the log if not? -fn worth_starting(store: &Arc, config: &MigrationConfig) -> bool { - if !config.enabled { - warn!( - "Storage migration is disabled. This node will keep reading both stores and \ - will never return the legacy environment's disk space." - ); - return false; - } - // The same question the spawn site asks. A different one here means a driver that is - // started and then returns immediately, which is how a node whose environment is a - // link to storage that was not mounted yet ends up never picking it up. - if !should_migrate(store) { - debug!("No legacy chunk environment; nothing to migrate"); - return false; - } - - let to_copy = store.legacy_only_keys().len(); - info!( - migration_event = "start", - to_copy, - legacy_bytes = store.legacy_bytes(), - "Storage migration starting: {to_copy} chunk(s) still only in the legacy \ - environment, {:.2} GiB to reclaim", - bytes_to_gib(store.legacy_bytes()) - ); - true -} - -/// Retry any removal that did not finish, and say whether the driver is done. -/// -/// Runs independently of the phase. A removal that could not finish leaves nothing to -/// migrate but a disk that has not come back, and the reasons it failed (a name already -/// taken, a directory that could not be flushed, a scanner holding a handle) are the kind -/// that clear on their own. -fn cleanup_state(store: &Arc) -> CleanupState { - if store.has_cleanup_pending() { - store.retry_cleanup(); - } - if store.has_cleanup_pending() || store.legacy_dir_is_on_disk() { - return CleanupState::Pending; - } - CleanupState::Finished -} - -/// Whether anything is left on disk for the driver to see through. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum CleanupState { - /// Something is still there. - Pending, - /// Nothing is. - Finished, -} - -/// Put a lost legacy handle back, if there is one to put back. -/// -/// A node that lost its handle to an environment still on disk cannot read the chunks that -/// live only there. The cause is usually transient, so this runs every tick rather than -/// leaving the node half-blind until somebody restarts it. -async fn maybe_recover_lost_handle(store: &Arc, next_attempt: &mut Instant) { - if Instant::now() < *next_attempt { - return; - } - if store.recover_lost_legacy_handle().await { - info!("Reopened the legacy chunk environment; the migration continues"); - *next_attempt = Instant::now(); - return; - } - // Backed off, because each attempt scans the whole environment and a cause that has - // not cleared in half a minute is unlikely to clear in the next. - *next_attempt = Instant::now() + HANDLE_RECOVERY_INTERVAL; -} - -/// How long this node has had the volume migration lock, and when it may ask again. -/// -/// Split out because the rule is easy to get wrong in one branch and not another: a -/// branch that takes the lock without recording when, or that gives it up without a -/// cooldown, silently opts out of the cap that stops one node holding a whole machine. -#[derive(Default)] -struct LockHold { - /// When the lock was taken, or when it was last used for real disk work. - since: Option, - /// Before this, do not ask for it again. - cooldown_until: Option, -} - -impl LockHold { - /// Record that the lock has just been taken. - fn taken(&mut self) { - self.since = Some(Instant::now()); - } - - /// Record that it is no longer held. - fn released(&mut self) { - self.since = None; - } - - /// Record that this tick used the lock for what it is for. - /// - /// Copying and verifying are the exclusive disk work the lock exists to serialise, so - /// time spent on them is not time spent sitting on it. Without this a node with a - /// large store would have the cap fire in the middle of a verification pass and - /// restart it, which is the cap causing the problem it prevents. - fn note_disk_work(&mut self) { - self.since = Some(Instant::now()); - } - - /// Has this node held the lock past the cap without using it? - fn has_overstayed(&self) -> bool { - self.since - .is_some_and(|at| at.elapsed() >= MAX_VOLUME_LOCK_HOLD) - } - - /// Give the lock up and stand back so somebody else can take it. - fn give_up(&mut self) { - self.since = None; - self.cooldown_until = Some(Instant::now() + VOLUME_LOCK_COOLDOWN); - } - - /// May this node ask for the lock yet? - fn may_ask(&self) -> bool { - !self - .cooldown_until - .is_some_and(|until| Instant::now() < until) - } -} - -/// What taking the volume lock produced for the driver loop. -enum LockStep { - /// Held, or not needed because none is possible here. - Proceed, - /// Someone else has it. Try again next tick. - WaitATick, -} - -/// Take the volume lock if it is not already held, stamping when it was taken. -/// -/// One place, because the stamp is what stops a node holding the volume against every -/// other node on the machine, and a branch that acquires without stamping silently opts -/// out of that. -fn take_volume_lock( - lock: &mut Option, - held: &mut LockHold, - root_dir: &Path, - scope: Option<&Path>, -) -> LockStep { - if lock.is_some() { - return LockStep::Proceed; - } - // After giving the volume up at the cap, stand back for a moment. Reacquiring in the - // same breath would hand nobody anything. - if !held.may_ask() { - return LockStep::WaitATick; - } - match VolumeLock::try_acquire(root_dir, scope) { - LockAttempt::Acquired(taken) => { - *lock = Some(taken); - held.taken(); - LockStep::Proceed - } - LockAttempt::Busy => { - debug!("Another node on this volume is migrating; waiting"); - LockStep::WaitATick - } - // No lock is possible here, so waiting for one would strand this node - // permanently. Proceed; the slack floor is the backstop. - LockAttempt::Unavailable => LockStep::Proceed, - } -} - -/// One pass of the copier. Returns `false` when this node cannot make progress that -/// needs the volume to itself. -async fn bridge_tick( - store: &Arc, - context: &MigrationContext, - config: &MigrationConfig, - next_shed_evaluation: &mut Instant, - shutdown: &CancellationToken, -) -> bool { - let remaining = store.legacy_only_keys(); - if remaining.is_empty() { - if let Err(e) = store.commit_to_files() { - // Not progress, and not something exclusive disk access fixes. Saying it was - // would reset the hold cap every tick and let this node keep the volume from - // every other node on the machine for as long as the failure lasts. - warn!( - "Everything is copied but the migration commitment could not be recorded: \ - {e}. Retrying on the next tick." - ); - return false; - } - return true; - } - - let ordered = rank_closest_first(remaining, context.self_xor); - let batch: Vec = ordered - .into_iter() - .take(config.batch_chunks.max(1)) - .collect(); - let report = match store - .copy_batch( - &batch, - config.copier_slack_bytes(), - config.copier_throttle_mib_per_sec, - shutdown, - ) - .await - { - Ok(report) => report, - Err(e) => { - // Still retried every tick, but the volume lock goes back: if this is - // permanent, holding it would block every other node on the volume on a node - // that is getting nowhere. - warn!("Storage migration copy failed: {e}. Retrying on the next tick."); - return false; - } - }; - - if report.copied > 0 { - debug!( - "Storage migration copied {} chunk(s) ({:.2} GiB) this pass", - report.copied, - bytes_to_gib(report.bytes) - ); - // A migration runs for hours. One periodic line at info level is what an operator - // watching a node actually sees, and what says the copier has not silently stalled. - let left = store.legacy_only_keys().len(); - if left % PROGRESS_LOG_EVERY < usize::try_from(report.copied).unwrap_or(usize::MAX) { - let no_rollback = store.writes_without_a_rollback_copy(); - info!( - migration_event = "progress", - remaining = left, - no_rollback_copy = no_rollback, - "Storage migration: {left} chunk(s) left to copy out of the legacy \ - environment, {no_rollback} write(s) made without a rollback copy" - ); - } - } - if report.unusable > 0 { - warn!( - "{} chunk(s) in the legacy environment did not match their own address and \ - were dropped from the key set", - report.unusable - ); - } - - if report.stopped_for_space { - // Out of space. Whatever happens next, this node is not going to write more until - // something changes, so it stops excluding its neighbours from the volume. That - // covers the 72-hour shed hold as well as an outright refusal: holding the lock - // for three days would leave every other node on the volume unmigrated. - if Instant::now() < *next_shed_evaluation { - return false; - } - *next_shed_evaluation = Instant::now() + SHED_REEVALUATION_INTERVAL; - return evaluate_shed(store, context, config).await; - } - true -} - -/// Decide whether the node may give up what it could not copy. -async fn evaluate_shed( - store: &Arc, - context: &MigrationContext, - config: &MigrationConfig, -) -> bool { - let remaining = store.legacy_only_keys(); - let short_by = remaining.len(); - - // Read from the one switch the auditors read, not from a second copy of it. There - // used to be two constants of the same name with two environment overrides, one on - // each side of this decision, and nothing coupling them: a node could have been - // willing to shed while every peer was still applying the full penalty, which is the - // exact outcome the release ordering exists to prevent. - if !crate::replication::config::close_group_storage_penalty_suspended() { - warn!( - "This node cannot fit {short_by} chunk(s) in the file store, but this release \ - has audit penalties switched back on, so giving anything up now would be \ - penalised by every peer. Keeping both stores. Add disk, or migrate this node \ - on a build that still suspends penalties." - ); - return false; - } - - if !config.allow_shed { - warn!( - "This node cannot fit {short_by} chunk(s) in the file store and shedding is \ - turned off. Add disk, or set storage.migration.allow_shed. Until then it \ - keeps serving from both stores and the legacy environment stays." - ); - return false; - } - - let state = store.migration_state(); - - // Wait for this node's turn. A close group is split into waves so at most - // CONCURRENT_MIGRATIONS_PER_GROUP of it are giving chunks up at once; if all seven - // holders went together, none could prove to the others that a copy survived and the - // whole group would sit deadlocked waiting on each other. Only nodes that have to give - // something up wait: a node with room has already copied everything and retired. - let wave = migration_wave_for(context.self_id.as_ref(), context.close_group_size); - if !wave_has_opened(&state, config, wave) { - info!( - "This node is {short_by} chunk(s) short of disk and is in migration wave {wave} \ - of {}. Its turn opens {} hour(s) after this build first started, so the rest of \ - its close group stays steady and can keep serving what it is about to give up.", - migration_wave_count(context.close_group_size), - config - .shed_hold_hours - .saturating_add(wave.saturating_mul(config.wave_hours)) - ); - return false; - } - - // Kept as its own check even though the wave now starts after it: the hold is about - // peers on an older build still applying the penalty, the wave is about the close - // group being able to cover for whoever moves. Different reasons, both required. - if !state.shed_hold_elapsed(config) { - info!( - "This node is {short_by} chunk(s) short of disk. Holding for {} hour(s) after \ - first start before giving any up, so peers still on an older build have \ - upgraded and stopped penalising a shed.", - config.shed_hold_hours - ); - return false; - } - - // First filter, and the cheap one: a node never gives up a chunk it is near the front - // of the group for. In practice it rarely fires, by construction, because the copier - // walks closest-first, so whatever is left when the disk fills is the far end of the - // list. Finding a protected key still uncopied means the node could not fit even the - // chunks it is closest to, which is exactly when it must not shed anything. - let mut protected = Vec::new(); - for key in &remaining { - if !context.may_shed(key).await { - protected.push(*key); - if protected.len() >= REFUSAL_SAMPLE { - break; - } - } - } - if !protected.is_empty() { - let sample: Vec = protected.iter().map(hex::encode).collect(); - warn!( - "This node is {short_by} chunk(s) short of disk, and at least {} of them are \ - chunks it is near the front of the group for (for example {}). It will not \ - give those up. The legacy environment stays and its disk is not returned \ - until storage is added.", - protected.len(), - sample.join(", ") - ); - return false; - } - - // Second filter, and the one that decides it: proof that somebody else holds every - // chunk this node is about to give up. Being far from a chunk is not evidence - // that a copy exists. During a fleet-wide migration the nodes that ought to hold it - // are exactly the ones that may also be out of disk, so the question has to be asked - // rather than inferred. - info!( - "Checking that other nodes hold the {short_by} chunk(s) this node cannot fit, \ - before giving any of them up" - ); - let unconfirmed = unconfirmed_by_neighbours(store, context, &remaining).await; - if !unconfirmed.is_empty() { - let sample: Vec = unconfirmed - .iter() - .take(REFUSAL_SAMPLE) - .map(hex::encode) - .collect(); - warn!( - "{} of the {short_by} chunk(s) this node cannot fit could not be proven to \ - exist anywhere else (for example {}). Nothing is given up and the legacy \ - environment stays. Add disk, or wait for replication to place them.", - unconfirmed.len(), - sample.join(", ") - ); - return false; - } - - info!( - migration_event = "shed", - shed = short_by, - "Every one of the {short_by} chunk(s) this node cannot fit is proven to be held \ - elsewhere. Committing to what it can hold. They stay readable from the legacy \ - environment until it is removed, and replication refetches whatever still belongs \ - here once there is room." - ); - if let Err(e) = store.commit_to_files() { - warn!("Could not record the migration commitment: {e}"); - return false; - } - true -} - -/// The keys still only in the legacy store that this node is too close to give up. -async fn keys_this_node_must_not_give_up( - store: &Arc, - context: &MigrationContext, -) -> Vec { - let mut must_keep = Vec::new(); - for key in store.legacy_only_keys() { - if !context.may_shed(&key).await { - must_keep.push(key); - } - } - must_keep -} - -/// Re-ask every network gate, after verification and immediately before the deletion. -/// -/// Verification re-reads the whole store and can run for hours. A gate satisfied before it -/// started says nothing about the moment of deletion: peers leave, replicas are pruned -/// elsewhere, and a write whose file half failed adds a fresh legacy-only key that has -/// faced none of these checks. This is the last point at which the answer can still be -/// acted on, so it is the point at which it has to be true. -async fn every_gate_still_holds( - store: &Arc, - context: &MigrationContext, - candidates: &std::collections::BTreeSet, -) -> Option { - // A key that joined the legacy-only set since the snapshot was taken has been through - // none of this. Stop now rather than asking the gates about a set that has already - // moved; the next tick copies it and takes a fresh snapshot. - let live: std::collections::BTreeSet = store.legacy_only_keys().into_iter().collect(); - if live != *candidates { - debug!( - "Legacy environment not retired: the set changed while the gates were being \ - checked ({} keys then, {} now)", - candidates.len(), - live.len() - ); - return Some(RetireOutcome::Waiting); - } - if !keys_this_node_must_not_give_up(store, context) - .await - .is_empty() - { - debug!("Legacy environment not retired: the shed rule changed during verification"); - return Some(RetireOutcome::Waiting); - } - if let Some(outcome) = shedding_is_still_safe(store, context, candidates).await { - return Some(outcome); - } - // And the retention contract once more, for the same reason. - if let Some(reason) = store.retirement_blocker(|k| context.still_answerable(k)) { - debug!("Legacy environment not retired: {reason}"); - return Some(RetireOutcome::Waiting); - } - None -} - -/// The last two questions before anything is deleted, asked in this order because the -/// order is the safety argument: reduce the claim, let the group learn it, then give the -/// chunks up. -/// -/// Returns `Some` with the reason to stop, or `None` when it is safe to proceed. -async fn shedding_is_still_safe( - store: &Arc, - context: &MigrationContext, - shedding: &std::collections::BTreeSet, -) -> Option { - // Nothing below is reached until the node has reduced its commitment (the phase - // is `Committed`) and that reduction has been rebuilt and published. What remains - // is to confirm the close group has actually *received* it, and that the chunks - // being given up still exist elsewhere. Only then is anything deleted. - if !shedding.is_empty() { - // A rotation is not the same as neighbours knowing. Until they have the - // smaller key set they keep auditing this node against the one it used to - // hold, and a wave of audit failures is as damaging as losing the chunks. - // Asked only when there is a commitment to deliver. A node whose file-backed set - // is empty commits to nothing, so there is no hash for a neighbour to acknowledge - // and this gate would never open, stranding its disk for good. Nothing is lost by - // skipping it: the gate exists to stop neighbours auditing this node against a key - // set it no longer holds, and a node claiming nothing cannot fail such an audit. - // The possession check below, which proves every chunk being given up still exists - // elsewhere, is the gate that protects the data, and it still runs. - let commits_to_nothing = store - .committable_keys() - .await - .is_ok_and(|keys| keys.is_empty()); - if commits_to_nothing { - info!( - "This node's file-backed set is empty, so it commits to nothing and has \ - no reduced commitment for its close group to receive. Proceeding to the \ - possession check on the {} chunk(s) it is giving up.", - shedding.len() - ); - } else if !context.neighbours_know_the_commitment().await { - info!( - "Holding: {} of this node's close group must receive its reduced \ - commitment before it gives up {} chunk(s). {} have it so far.", - context.commitment_recipients_needed(), - shedding.len(), - context - .commitment - .as_ref() - .map_or(0, |s| s.current_delivered_peer_count()) - ); - // Give the volume back while waiting on this. It is a network condition, not - // a disk one, and it may never resolve: holding the lock through it would let - // one node stop every other node on the machine from ever starting. - return Some(RetireOutcome::NoWorkToSerialise); - } - // Asked again here, not only when the node committed. Hours pass in between, - // the group moves, and a peer that held a copy then may not now. This is the - // last moment at which the answer still matters. - let ordered: Vec = shedding.iter().copied().collect(); - let unconfirmed = unconfirmed_by_neighbours(store, context, &ordered).await; - if !unconfirmed.is_empty() { - let sample: Vec = unconfirmed - .iter() - .take(REFUSAL_SAMPLE) - .map(hex::encode) - .collect(); - warn!( - "{} of the {} chunk(s) this node is giving up can no longer be proven \ - to exist elsewhere (for example {}). The legacy environment stays.", - unconfirmed.len(), - shedding.len(), - sample.join(", ") - ); - return Some(RetireOutcome::NoWorkToSerialise); - } - } - None -} - -/// What one pass of the retirement gate concluded. -enum RetireOutcome { - /// The legacy environment is gone. The driver is finished. - Done, - /// Exclusive disk work happened this tick: copying, or re-reading the store to verify - /// it. Keep the volume, and count the time as time spent using it rather than time - /// spent holding it. - Working, - /// Still working towards it, but waiting on a clock rather than on the disk. Keep the - /// volume, because the node is about to need it, but let the hold cap run. - Waiting, - /// Blocked on something no amount of exclusive disk access will fix. - NoWorkToSerialise, -} - -/// When this node last said out loud that its migration needs a person. -static LAST_OPERATOR_WARNING: parking_lot::Mutex> = parking_lot::Mutex::new(None); - -/// How often to repeat it. Often enough to be noticed, rarely enough not to drown the log. -const OPERATOR_WARNING_INTERVAL: Duration = Duration::from_secs(3600); - -/// Should the "this needs a person" warning be repeated now? -/// -/// The condition it reports is checked on every tick and does not clear on its own, so -/// without this it would be a line every thirty seconds for as long as the node runs. -fn operator_should_hear_again() -> bool { - let mut last = LAST_OPERATOR_WARNING.lock(); - let now = Instant::now(); - if last.is_some_and(|at| now.duration_since(at) < OPERATOR_WARNING_INTERVAL) { - return false; - } - *last = Some(now); - true -} - -/// The retention contract, asked before anything else in the tick. -/// -/// `Some` with what the driver should do, or `None` when nothing is in the way. -fn blocked_before_the_gates( - store: &Arc, - context: &MigrationContext, - config: &MigrationConfig, -) -> Option { - let reason = store.retirement_blocker(|k| context.still_answerable(k))?; - // Three blockers no amount of exclusive disk access will clear: an environment this - // node cannot read, one it cannot classify at all, and one it must not delete because - // it is a link to somewhere else. Each needs a person, so give the volume back to the - // nodes that can use it and say so where an operator will see it rather than at debug. - if store.has_lost_its_legacy_handle() - || store.legacy_cannot_be_classified() - || store.legacy_is_a_link() - { - if operator_should_hear_again() { - warn!( - migration_event = "needs_an_operator", - "The legacy chunk environment will not be retired automatically: {reason}" - ); - } - return Some(RetireOutcome::NoWorkToSerialise); - } - debug!("Legacy environment not retired yet: {reason}"); - Some(if config.retire_legacy { - RetireOutcome::Waiting - } else { - // Retirement is switched off on this node, so it will never free its disk here - // however long it waits. - RetireOutcome::NoWorkToSerialise - }) -} - -/// One pass of the retirement gate. -async fn retire_tick( - store: &Arc, - context: &MigrationContext, - config: &MigrationConfig, - verified: &mut Option<(VerifyReport, Instant)>, - shutdown: &CancellationToken, -) -> RetireOutcome { - if let Some(outcome) = blocked_before_the_gates(store, context, config) { - return outcome; - } - - // Re-check the shed rule against live routing immediately before the destructive - // step, not once when the node committed hours ago. Two things put a key back into - // the legacy-only set after that decision: a file that failed verification and is now - // served from the legacy copy, and a write whose file half failed. Neither went - // through the rank check, and both would be thrown away by the removal below. - let must_keep = keys_this_node_must_not_give_up(store, context).await; - if !must_keep.is_empty() { - warn!( - "{} chunk(s) are still only in the legacy environment and this node is too \ - close to them to give them up. Copying them before anything is removed.", - must_keep.len() - ); - match store - .copy_batch( - &must_keep, - config.copier_slack_bytes(), - config.copier_throttle_mib_per_sec, - shutdown, - ) - .await - { - Ok(report) if report.stopped_for_space => { - warn!( - "Out of disk while copying {} chunk(s) this node must not give up. \ - The legacy environment stays until there is room for them.", - must_keep.len() - ); - *verified = None; - return RetireOutcome::NoWorkToSerialise; - } - Ok(_) => {} - Err(e) => { - warn!("Could not copy the chunks this node must keep: {e}"); - *verified = None; - return RetireOutcome::NoWorkToSerialise; - } - } - // Anything copied changed the file store, so a previous verification no longer - // covers it. - *verified = None; - return RetireOutcome::Working; - } - - // The real report from a recent pass, never a fabricated one. Reuse deliberately does - // NOT refresh the window: re-arming it from a reused proof would let a node that - // keeps deferring retirement run the verification exactly once and coast on it. - let reusable = verified - .filter(|(_, at)| at.elapsed() < VERIFICATION_REUSE_WINDOW) - .map(|(proof, _)| proof); - let proof = match reusable { - Some(proof) => proof, - None => match store - .verify_before_retire(config.copier_throttle_mib_per_sec, shutdown) - .await - { - Ok(proof) => { - if proof.is_clean() { - *verified = Some((proof, Instant::now())); - } - proof - } - Err(e) => { - // A pass that failed is not progress, however quickly it failed, and - // treating it as work would let a node whose store cannot be read hold - // the volume against every other node on the machine for good. - warn!("Pre-retirement verification failed: {e}. Retrying on the next tick."); - return RetireOutcome::NoWorkToSerialise; - } - }, - }; - if !proof.is_clean() { - *verified = None; - warn!( - "Pre-retirement verification found {} chunk(s) that are damaged in the file \ - store and cannot be repaired from the legacy environment. The legacy \ - environment stays.", - proof.unrepairable() - ); - return RetireOutcome::NoWorkToSerialise; - } - - // Snapshotted BEFORE the gates, not after. Every gate below is asked about exactly - // this set, and exactly this set is what the removal is permitted to destroy. Taken - // afterwards, a key that joined between the last gate and the snapshot would be - // counted as approved having passed nothing, which is the case the gates exist for. - let approved: std::collections::BTreeSet = - store.legacy_only_keys().into_iter().collect(); - - if let Some(outcome) = every_gate_still_holds(store, context, &approved).await { - return outcome; - } - - let kept = store.current_chunks().unwrap_or(0); - let shed = store.migration_state().shed_key_count; - match store - .retire_legacy( - &proof, - &|k: &XorName| context.still_answerable(k), - &approved, - ) - .await - { - Ok(freed) => { - log_migration_complete(kept, shed, freed); - RetireOutcome::Done - } - Err(e) => { - debug!("Legacy environment not retired yet: {e}"); - RetireOutcome::Waiting - } - } -} - -/// Convert a byte count to GiB for human-readable log messages. -#[allow(clippy::cast_precision_loss)] // display only -#[cfg_attr(not(feature = "logging"), allow(dead_code))] -fn bytes_to_gib(bytes: u64) -> f64 { - bytes as f64 / (1024.0 * 1024.0 * 1024.0) -} - -#[cfg(test)] -#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] -mod tests { - use super::*; - use serial_test::serial; - use tempfile::TempDir; - - fn peer_id(byte: u8) -> PeerId { - let mut bytes = [0u8; 32]; - if let Some(slot) = bytes.first_mut() { - *slot = byte; - } - PeerId::from_bytes(bytes) - } - - /// The width shedding is measured against: the admission group, not the close group. - const WIDTH: usize = storage_admission_width(7); - - #[test] - fn shedding_is_measured_against_the_width_the_pruner_protects() { - // The pruner treats the admission group (close group plus its margin) as strictly - // in-range and refuses to delete inside it. A one-off migration must not be more - // willing to drop a chunk than the thing that runs every day. - assert_eq!(WIDTH, storage_admission_width(7)); - assert!( - storage_admission_width(7) > 7, - "the admission group is wider than the close group" - ); - // A rank the close group would have called sheddable is protected here. - assert!(!rank_is_sheddable(GroupRank::Inside(5), WIDTH)); - assert!(!rank_is_sheddable(GroupRank::Inside(6), WIDTH)); - } - - #[test] - fn a_node_never_gives_up_a_chunk_it_is_among_the_closest_to() { - // This node is one of the closest for these ranks. Giving one of them up is the - // case where every short-of-space holder could drop the same chunk and take its - // last replica, so it is refused outright. - for rank in 0..WIDTH - SHEDDABLE_TAIL_RANKS { - assert!( - !rank_is_sheddable(GroupRank::Inside(rank), WIDTH), - "rank {rank} must be protected" - ); - } - // The last two positions may be given up: a chunk has exactly one holder at each, - // so it is only ever a candidate for two of its holders. - for rank in WIDTH - SHEDDABLE_TAIL_RANKS..WIDTH { - assert!( - rank_is_sheddable(GroupRank::Inside(rank), WIDTH), - "rank {rank} is in the tail and may be shed" - ); - } - // Out of range entirely: nothing to protect. - assert!(rank_is_sheddable(GroupRank::Outside, WIDTH)); - // No routing to consult is never a licence. - assert!(!rank_is_sheddable(GroupRank::Unknown, WIDTH)); - } - - #[test] - fn a_group_narrower_than_the_tail_protects_everything_in_it() { - // A group with no tail has nothing to give up. Subtracting saturatingly would put - // the threshold at zero and make every member sheddable, which is exactly - // backwards for the narrowest groups. - assert!(!rank_is_sheddable(GroupRank::Inside(0), 2)); - assert!(!rank_is_sheddable(GroupRank::Inside(0), 1)); - assert!(!rank_is_sheddable(GroupRank::Inside(1), 2)); - // Out of the group entirely is still out. - assert!(rank_is_sheddable(GroupRank::Outside, 2)); - // And a group with a tail still has one. - assert!(!rank_is_sheddable(GroupRank::Inside(0), 3)); - assert!(rank_is_sheddable(GroupRank::Inside(1), 3)); - } - - #[test] - fn the_marker_round_trips_and_a_corrupt_one_starts_over_conservatively() { - let dir = TempDir::new().expect("temp dir"); - let mut state = MigrationState::new(MigrationPhase::Bridging); - state.phase = MigrationPhase::Committed; - state.shed_key_count = 12; - state.committed_at_unix = Some(1_700_000_000); - state.save(dir.path()).expect("save"); - - let loaded = MigrationState::load_or_new(dir.path(), MigrationPhase::Bridging); - assert_eq!(loaded.phase, MigrationPhase::Committed); - assert_eq!(loaded.shed_key_count, 12); - assert_eq!(loaded.committed_at_unix, Some(1_700_000_000)); - - // An unreadable marker restarts the clocks rather than being fatal. Losing it - // delays a migration and can never rush one. - std::fs::write(state_path(dir.path()), b"not json").expect("corrupt"); - let recovered = MigrationState::load_or_new(dir.path(), MigrationPhase::Bridging); - assert_eq!(recovered.phase, MigrationPhase::Bridging); - assert_eq!(recovered.committed_at_unix, None); - } - - #[test] - fn the_shed_hold_and_retirement_clocks_run_from_recorded_times() { - let config = MigrationConfig { - shed_hold_hours: 72, - retire_delay_hours: MIN_RETIRE_DELAY_HOURS, - ..MigrationConfig::default() - }; - - let mut state = MigrationState::new(MigrationPhase::Bridging); - assert!(!state.shed_hold_elapsed(&config), "just started"); - state.first_start_unix = now_unix().saturating_sub(73 * 3600); - assert!(state.shed_hold_elapsed(&config)); - - assert!( - !state.retire_delay_elapsed(&config), - "never committed, so the clock has not started" - ); - state.committed_at_unix = Some(now_unix().saturating_sub(3600)); - assert!(!state.retire_delay_elapsed(&config), "an hour is not four"); - state.committed_at_unix = - Some(now_unix().saturating_sub(MIN_RETIRE_DELAY_HOURS * 3600 + 60)); - assert!(state.retire_delay_elapsed(&config)); - } - - #[test] - fn only_one_node_on_a_volume_migrates_at_a_time() { - let volume = TempDir::new().expect("temp dir"); - let node_a = volume.path().join("node-a"); - let node_b = volume.path().join("node-b"); - std::fs::create_dir_all(&node_a).expect("mkdir"); - std::fs::create_dir_all(&node_b).expect("mkdir"); - - let LockAttempt::Acquired(held) = VolumeLock::try_acquire(&node_a, None) else { - panic!("the first node must take the lock"); - }; - assert!( - matches!(VolumeLock::try_acquire(&node_b, None), LockAttempt::Busy), - "a second node on the same volume must be told to wait, not that no lock exists" - ); - - drop(held); - assert!( - matches!( - VolumeLock::try_acquire(&node_b, None), - LockAttempt::Acquired(_) - ), - "and take it once the first is done" - ); - } - - /// Seed a real LMDB chunk store, the way a node upgrading into this build has one. - async fn seed_legacy(root: &std::path::Path, count: u32) -> Vec { - let lmdb = crate::storage::LmdbStorage::new(crate::storage::LmdbStorageConfig { - root_dir: root.to_path_buf(), - verify_on_read: true, - max_map_size: 0, - disk_reserve: 0, - }) - .await - .expect("open legacy"); - let mut keys = Vec::new(); - for i in 0..count { - let content = format!("legacy-chunk-{i}").into_bytes(); - let addr = crate::client::compute_address(&content); - lmdb.put(&addr, &content).await.expect("legacy put"); - keys.push(addr); - } - lmdb.wait_idle().await; - drop(lmdb); - keys - } - - /// The whole point, end to end: a node that starts with an LMDB chunk store and a - /// disk to hold it finishes with the chunks in files and the LMDB gone. - /// - /// Driven by `run`, the same entry point node startup calls, rather than by poking the - /// pieces. That matters: the wiring that calls it went missing once and every test - /// passed, because they all built the store directly and a node with no legacy store - /// starts no migration. - #[tokio::test] - async fn a_node_with_room_copies_everything_and_removes_the_legacy_store() { - const CHUNKS: u32 = 24; - - let tmp = TempDir::new().expect("temp dir"); - // Nested, so the volume lock this node takes lives in its own directory rather - // than one shared with every other test running in parallel. - let root = tmp.path().join("node"); - std::fs::create_dir_all(&root).expect("mkdir"); - let keys = seed_legacy(&root, CHUNKS).await; - - let mut config = crate::storage::ChunkStoreConfig { - root_dir: root.clone(), - ..crate::storage::ChunkStoreConfig::test_default() - }; - // Deliberately NOT setting `retire_legacy`. The shipped default has to be what - // carries this all the way to a removed environment, or the release migrates every - // node and reclaims nothing. - config.migration.tick_secs = 1; - // Scoped to this test's own directory. In production the lock is keyed by the - // filesystem, so without this every test on this machine would serialise against - // every other one that runs a migration. - config.migration.lock_dir = Some(root.clone()); - config.migration.copier_throttle_mib_per_sec = 0; - let store = Arc::new( - crate::storage::ChunkStore::new(config) - .await - .expect("open store"), - ); - - // Precondition: everything is in the legacy store and nothing is in files. - assert!(store.has_legacy(), "the node must start with an LMDB store"); - assert_eq!(store.migration_phase(), MigrationPhase::Bridging); - assert_eq!(store.legacy_only_keys().len(), CHUNKS as usize); - assert!(root.join("chunks.mdb").exists()); - - let shutdown = CancellationToken::new(); - let driver = tokio::spawn(run( - Arc::clone(&store), - MigrationContext { - p2p: None, - self_id: None, - self_xor: None, - commitment: None, - replication: None, - sync_state: None, - audit_challenge_coordinator: None, - peer_commitments: None, - close_group_size: 7, - }, - shutdown.clone(), - )); - - // The copier runs on its own and settles once nothing is left only in the legacy - // store. This node has room, so it sheds nothing and needs no network at all. - wait_for( - &store, - MigrationPhase::Committed, - "the copier should finish", - ) - .await; - assert!( - store.legacy_only_keys().is_empty(), - "every chunk should have been copied" - ); - - // Stand in for the commitment builder, which lives in the replication engine: the - // retirement gate wants the reduced commitment published and its window elapsed. - store.note_commitment_rebuilt(); - store.note_commitment_rebuilt(); - store.force_migration_state(|s| { - s.committed_at_unix = - Some(now_unix().saturating_sub(MIN_RETIRE_DELAY_HOURS * 3600 + 60)); - }); - - wait_for( - &store, - MigrationPhase::FilesOnly, - "retirement should complete", - ) - .await; - shutdown.cancel(); - let _ = tokio::time::timeout(Duration::from_secs(10), driver).await; - - // The point of the whole exercise: the LMDB is gone from the filesystem. - assert!( - !root.join("chunks.mdb").exists(), - "the legacy store must be removed, which is the only moment disk comes back" - ); - assert!(!store.has_legacy()); - - // And nothing was lost: every chunk still reads, now out of a file. - assert_eq!(store.current_chunks().expect("count"), u64::from(CHUNKS)); - for (i, key) in keys.iter().enumerate() { - let expected = format!("legacy-chunk-{i}").into_bytes(); - assert_eq!( - store.get(key).await.expect("get").expect("present"), - expected, - "chunk {i} did not survive the migration" - ); - } - - // In files, under the suffix shard its address names. - let sample = keys.first().copied().expect("a key"); - let path = root - .join(crate::storage::file_store::CHUNKS_DIR_NAME) - .join(format!("{:02x}", sample.last().copied().unwrap_or(0))) - .join(hex::encode(sample)); - assert!(path.exists(), "expected a chunk file at {}", path.display()); - } - - /// The other half: a node that cannot fit its chunks and cannot prove anyone else - /// holds them keeps both stores and deletes nothing. - /// - /// This is the case that must fail safe. The node is out of disk, so it would like to - /// give chunks up, but with no view of the network it cannot show a single one exists - /// elsewhere. Refusing costs it disk. Proceeding would cost the network data. - #[tokio::test] - async fn a_node_that_cannot_prove_its_chunks_are_safe_deletes_nothing() { - const CHUNKS: u32 = 8; - - let tmp = TempDir::new().expect("temp dir"); - let root = tmp.path().join("node"); - std::fs::create_dir_all(&root).expect("mkdir"); - let keys = seed_legacy(&root, CHUNKS).await; - - let mut config = crate::storage::ChunkStoreConfig { - root_dir: root.clone(), - // Nothing will fit: the copier stops for space on its first chunk. - disk_reserve: u64::MAX / 2, - ..crate::storage::ChunkStoreConfig::test_default() - }; - config.migration.retire_legacy = true; - config.migration.tick_secs = 1; - // Scoped to this test's own directory. In production the lock is keyed by the - // filesystem, so without this every test on this machine would serialise against - // every other one that runs a migration. - config.migration.lock_dir = Some(root.clone()); - // Elapsed, so the hold is not what is doing the refusing here. - config.migration.shed_hold_hours = 0; - config.migration.wave_hours = 0; - let store = Arc::new( - crate::storage::ChunkStore::new(config) - .await - .expect("open store"), - ); - assert_eq!(store.legacy_only_keys().len(), CHUNKS as usize); - - let shutdown = CancellationToken::new(); - let driver = tokio::spawn(run( - Arc::clone(&store), - MigrationContext { - p2p: None, - self_id: None, - self_xor: None, - commitment: None, - replication: None, - sync_state: None, - audit_challenge_coordinator: None, - peer_commitments: None, - close_group_size: 7, - }, - shutdown.clone(), - )); - - // Give it long enough to have tried, re-tried, and evaluated shedding. - tokio::time::sleep(Duration::from_secs(5)).await; - shutdown.cancel(); - let _ = tokio::time::timeout(Duration::from_secs(10), driver).await; - - assert_eq!( - store.migration_phase(), - MigrationPhase::Bridging, - "a node that cannot prove its chunks are held elsewhere must not commit" - ); - assert!( - root.join("chunks.mdb").exists(), - "and must not remove the only copy of them" - ); - assert_eq!(store.legacy_only_keys().len(), CHUNKS as usize); - for (i, key) in keys.iter().enumerate() { - assert_eq!( - store.get(key).await.expect("get").expect("present"), - format!("legacy-chunk-{i}").into_bytes(), - "chunk {i} must still be served throughout" - ); - } - } - - /// Poll until the store reaches `phase`, or fail with what it reached instead. - /// - /// The deadline is generous because it is measured on the wall clock while the driver - /// it is waiting on runs on the runtime. On a saturated machine both stretch, and a - /// deadline sized for the work rather than for the contention turns a slow build into - /// a failing test. The work itself is two ticks. - async fn wait_for(store: &Arc, phase: MigrationPhase, what: &str) { - let deadline = std::time::Instant::now() + Duration::from_secs(180); - while std::time::Instant::now() < deadline { - if store.migration_phase() == phase { - return; - } - tokio::time::sleep(Duration::from_millis(100)).await; - } - panic!( - "{what}: still in {:?} after the deadline, expected {phase:?}", - store.migration_phase() - ); - } - - /// A marker that cannot be used is replaced on disk, not just in memory. - /// - /// The shed hold counts from `first_start_unix`, and the whole reason this is written - /// at startup rather than at the first phase change is that a marker written later - /// would restart that clock on every reboot. A marker that is present but unusable used - /// to defeat that: it was replaced in memory and left on disk, so every start read the - /// same bad file and stamped a fresh clock. A node restarting more often than the hold - /// would then never become eligible to shed and never finish migrating, which is the - /// failure the doc comment on `load_or_create` names. - #[test] - fn an_unusable_marker_is_replaced_on_disk_so_the_hold_does_not_restart() { - for (name, bad) in [ - ("truncated", br#"{"schema":1,"phase":"brid"#.to_vec()), - ("not json at all", b"\x00\x01\x02".to_vec()), - ( - "a newer schema", - serde_json::json!({ - "schema": 9_999, - "phase": "bridging", - "first_start_unix": 1, - "committed_at_unix": null, - "rebuilds_since_commit": 0, - "shed_key_count": 0, - "kept_key_count": 0, - }) - .to_string() - .into_bytes(), - ), - // The one most worth keeping, and the one a parse into today's struct cannot - // read: a phase this build has no name for, and a field it does not know. - ( - "a newer schema this build cannot parse at all", - serde_json::json!({ - "schema": 9_999, - "phase": "some_phase_from_the_future", - "first_start_unix": 1, - "something_this_build_never_heard_of": {"a": 1}, - }) - .to_string() - .into_bytes(), - ), - ] { - let dir = TempDir::new().expect("temp dir"); - std::fs::write(state_path(dir.path()), &bad).expect("plant the bad marker"); - - let first = MigrationState::load_or_create(dir.path(), MigrationPhase::Bridging); - let written = std::fs::read(state_path(dir.path())).expect("read it back"); - assert_ne!(written, bad, "{name}: the unusable marker was left on disk"); - if name.starts_with("a newer schema") { - // Moved aside rather than destroyed: it was written on purpose by a build - // that knew more than this one. - let mut kept = state_path(dir.path()).into_os_string(); - kept.push(".schema-9999"); - let kept = PathBuf::from(kept); - assert_eq!( - std::fs::read(&kept).expect("the newer marker must be kept"), - bad, - "the newer marker was written over rather than set aside" - ); - } - - // And the clock survives the next start, which is the point of all this. - let second = MigrationState::load_or_create(dir.path(), MigrationPhase::Bridging); - assert_eq!( - second.first_start_unix, first.first_start_unix, - "{name}: the shed hold restarted on the next boot" - ); - } - } - - /// A marker whose clock is impossible is corrected on disk too. - /// - /// `with_sane_clocks` fixes a future-dated marker for the process that read it. Left - /// there, the same correction is made again on every start, from a new now each time, - /// which is the same repeating reset by another route. - #[test] - fn a_future_dated_marker_is_corrected_on_disk_not_only_in_memory() { - let dir = TempDir::new().expect("temp dir"); - let ahead = now_unix().saturating_add(60 * 60 * 24 * 365); - let planted = serde_json::json!({ - "schema": 1, - "phase": "bridging", - "first_start_unix": ahead, - "committed_at_unix": null, - "rebuilds_since_commit": 0, - "shed_key_count": 0, - "kept_key_count": 0, - }) - .to_string(); - std::fs::write(state_path(dir.path()), &planted).expect("plant it"); - - let first = MigrationState::load_or_create(dir.path(), MigrationPhase::Bridging); - assert!( - first.first_start_unix < ahead, - "the clock should have been brought back to something possible" - ); - - let reread: MigrationState = - serde_json::from_slice(&std::fs::read(state_path(dir.path())).expect("read")) - .expect("the marker on disk must now parse"); - assert_eq!( - reread.first_start_unix, first.first_start_unix, - "the correction was made in memory and not written back" - ); - } - - /// A marker that is already right is not rewritten on every start. - /// - /// The counterpart to the two above: persisting on disagreement must not turn into - /// persisting unconditionally, which would put a write on every node's start path for - /// nothing. - #[test] - fn a_usable_marker_is_left_exactly_as_it_is() { - let dir = TempDir::new().expect("temp dir"); - let first = MigrationState::load_or_create(dir.path(), MigrationPhase::Bridging); - - // Written back out by hand in a shape this build would never produce: the same - // facts, different spacing and key order. Comparing the bytes of a marker this - // build wrote against the bytes after a second start proves nothing, because an - // unconditional rewrite produces the same bytes and the test passes either way. - // Something semantically equal but textually different is the only thing that can - // tell "left alone" from "written again". - let noncanonical = format!( - "{{\"kept_key_count\":{},\"shed_key_count\":{},\"rebuilds_since_commit\":{},\ - \"committed_at_unix\":null,\"first_start_unix\":{},\"phase\":\"bridging\",\ - \"schema\":{}}}", - first.kept_key_count, - first.shed_key_count, - first.rebuilds_since_commit, - first.first_start_unix, - first.schema - ); - std::fs::write(state_path(dir.path()), &noncanonical).expect("write"); - - MigrationState::load_or_create(dir.path(), MigrationPhase::Bridging); - assert_eq!( - std::fs::read_to_string(state_path(dir.path())).expect("read"), - noncanonical, - "a marker that already said the right thing was rewritten for no reason" - ); - } - - /// Two nodes on one disk take the same lock, and the override is what makes that true - /// where they do not share a `/tmp`. - /// - /// `lock_path_for` had no coverage at all, which is how it came to be right in a way - /// that is false on our own fleet: every shipped test sets `lock_dir` and so never - /// calls it. What the lock is for is one node copying at a time on a shared disk, so - /// what has to be true is that two different node roots on one volume produce one path. - /// - /// Nothing here touches process-wide environment. An earlier version of this test set - /// `TMPDIR` to stage the private-`/tmp` case, which moved every other test's temporary - /// directory and then deleted it underneath them: sixty-three unrelated tests failed - /// with LMDB errors that pointed nowhere near the cause. - #[test] - fn nodes_on_one_volume_agree_on_a_lock_path_when_they_are_told_where_it_is() { - let volume = TempDir::new().expect("temp dir"); - let a = volume.path().join("node-0"); - let b = volume.path().join("node-1"); - std::fs::create_dir_all(&a).expect("mkdir"); - std::fs::create_dir_all(&b).expect("mkdir"); - - // With no override and one shared temporary directory, which is the case the - // default is right for: two roots on one volume, one lock. - assert_eq!( - lock_path_with(&a, None), - lock_path_with(&b, None), - "two roots on one volume must share a lock when they share a /tmp" - ); - - // Where they do not share one, the default gives each node a lock of its own and - // every one of them takes it. That is the deployment hazard, and it is why the - // override exists rather than something the code can detect. - assert_ne!( - lock_path_with(&a, Some("/tmp/private-to-node-0")), - lock_path_with(&b, Some("/tmp/private-to-node-1")), - "different lock directories must give different locks, or the override would \ - not be able to express anything" - ); - - // And told where it lives, both land on it whatever their own root is. - let told = volume.path().to_string_lossy().into_owned(); - let shared_a = lock_path_with(&a, Some(&told)); - let shared_b = lock_path_with(&b, Some(&told)); - assert_eq!( - shared_a, shared_b, - "nodes told where the lock lives must all use it" - ); - assert!(shared_a.starts_with(volume.path()), "and use the one named"); - assert_eq!( - lock_path_with(&a, Some(" ")), - lock_path_with(&a, None), - "an empty setting is not a location and must not be treated as one" - ); - - // And the lock itself then does its job across the two roots. - let LockAttempt::Acquired(held) = VolumeLock::try_acquire(&a, Some(volume.path())) else { - panic!("the first node must take it"); - }; - assert!( - matches!( - VolumeLock::try_acquire(&b, Some(volume.path())), - LockAttempt::Busy - ), - "the second must wait rather than copy alongside it" - ); - drop(held); - assert!(matches!( - VolumeLock::try_acquire(&b, Some(volume.path())), - LockAttempt::Acquired(_) - )); - } - - /// A lock directory that cannot be written is reported, not silently ignored. - /// - /// The override is a deployment fact, so a typo in it must not read as "no lock needed - /// here". `Unavailable` is the honest answer and it already warns; what this pins is - /// that a bad override does not quietly fall back to a path that would appear to work. - #[test] - fn a_lock_directory_that_does_not_exist_is_unavailable_rather_than_ignored() { - let dir = TempDir::new().expect("temp dir"); - let missing = dir.path().join("no-such-directory"); - assert!(matches!( - VolumeLock::try_acquire(dir.path(), Some(&missing)), - LockAttempt::Unavailable - )); - } - - #[tokio::test] - async fn the_driver_exits_immediately_when_there_is_nothing_to_migrate() { - // The whole feature hangs off `run` being reachable from node startup. A port that - // dropped that call once already, and nothing caught it, because a fresh node has - // no legacy environment and every test built one directly. This asserts the entry - // point is callable and terminates on its own for a node with nothing to do. - let dir = TempDir::new().expect("temp dir"); - let store = Arc::new( - crate::storage::ChunkStore::new(crate::storage::ChunkStoreConfig { - root_dir: dir.path().to_path_buf(), - ..crate::storage::ChunkStoreConfig::test_default() - }) - .await - .expect("open store"), - ); - assert!(!store.has_legacy()); - - let context = MigrationContext { - p2p: None, - self_id: None, - self_xor: None, - commitment: None, - replication: None, - sync_state: None, - audit_challenge_coordinator: None, - peer_commitments: None, - close_group_size: 7, - }; - tokio::time::timeout( - Duration::from_secs(5), - run(store, context, CancellationToken::new()), - ) - .await - .expect("the driver must return rather than idle when there is nothing to migrate"); - } - - #[tokio::test] - async fn a_node_with_no_view_of_the_network_gives_up_nothing() { - // Every field is `None`, which is what a devnet or a node whose routing is not up - // yet looks like. No view of the network is no evidence, and the answer has to be - // "keep everything" rather than "nobody is closer, so give it all away". - let dir = TempDir::new().expect("temp dir"); - let store = Arc::new( - crate::storage::ChunkStore::new(crate::storage::ChunkStoreConfig { - root_dir: dir.path().to_path_buf(), - ..crate::storage::ChunkStoreConfig::test_default() - }) - .await - .expect("open store"), - ); - let context = MigrationContext { - p2p: None, - self_id: None, - self_xor: None, - commitment: None, - replication: None, - sync_state: None, - audit_challenge_coordinator: None, - peer_commitments: None, - close_group_size: 7, - }; - - let keys = vec![[1u8; 32], [2u8; 32], [3u8; 32]]; - let unconfirmed = unconfirmed_by_neighbours(&store, &context, &keys).await; - assert_eq!( - unconfirmed, keys, - "with no routing state every key must count as unproven" - ); - for key in &keys { - assert!( - !context.may_shed(key).await, - "and none of them may be given up" - ); - } - } - - #[test] - fn the_possession_threshold_comes_from_the_whole_group_not_the_qualifying_subset() { - use crate::replication::pruning::{prune_proofs_needed, target_peers_reported_present}; - use std::collections::{HashMap, HashSet}; - - // Seven holders. Deriving the bar from whichever peers happen to qualify is how - // two last holders destroy a chunk between them: each sees only the other - // publishing, so each needs exactly one proof, each gets it from the other, and - // both delete. The bar must come from the group. - let key = [7u8; 32]; - let group: Vec = (0..6u8).map(peer_id).collect(); - let only_one_qualifies: Vec = group.iter().take(1).copied().collect(); - - // That one peer does answer the challenge. - let mut proofs: HashMap> = HashMap::new(); - proofs.insert(key, only_one_qualifies.iter().copied().collect()); - - // The dangerous reading: bar taken from the qualifying subset, so one is enough. - assert!( - target_peers_reported_present( - &key, - &only_one_qualifies, - &proofs, - prune_proofs_needed(only_one_qualifies.len()), - ), - "this is the mistake being guarded against, shown here to be a real risk" - ); - - // The correct reading: bar taken from the whole group, so one is nowhere near. - assert!( - !target_peers_reported_present( - &key, - &only_one_qualifies, - &proofs, - prune_proofs_needed(group.len()), - ), - "one proof must never satisfy a group of six" - ); - - // And with the whole group answering, it passes. - proofs.insert(key, group.iter().copied().collect()); - assert!(target_peers_reported_present( - &key, - &group, - &proofs, - prune_proofs_needed(group.len()), - )); - } - - #[test] - #[serial] - fn shedding_reads_the_same_switch_the_auditors_read() { - use crate::replication::config::{ - close_group_storage_penalty_suspended, set_close_group_storage_penalty_suspended, - }; - // One switch, not two. There used to be a second constant of the same name with - // its own environment override on this side of the decision, and nothing coupling - // them: a node could have been willing to shed while every peer still applied the - // full penalty, which is precisely what the release ordering exists to prevent. - set_close_group_storage_penalty_suspended(true); - assert!(close_group_storage_penalty_suspended()); - set_close_group_storage_penalty_suspended(false); - assert!(!close_group_storage_penalty_suspended()); - set_close_group_storage_penalty_suspended( - crate::replication::config::RELEASE_SUSPEND_CLOSE_GROUP_STORAGE_PENALTY, - ); - } - - #[test] - fn a_close_group_is_split_into_enough_waves_to_bound_concurrent_migrations() { - // Seven holders, two at a time, is four waves. - assert_eq!(migration_wave_count(7), 4); - assert_eq!(migration_wave_count(2), 1); - assert_eq!(migration_wave_count(1), 1); - // A degenerate width must still yield a usable wave count rather than dividing by - // zero or collapsing to "everyone at once". - assert_eq!(migration_wave_count(0), 1); - } - - #[test] - fn wave_assignment_is_stable_per_node_and_spread_across_the_group() { - use std::collections::HashMap; - let waves = migration_wave_count(7); - - // The same node always gets the same wave: a restart must not move a node into a - // turn that has already passed. - let peer = peer_id(7); - assert_eq!( - migration_wave_for(Some(&peer), 7), - migration_wave_for(Some(&peer), 7) - ); - - // And across many nodes every wave is used, so the group is genuinely staggered - // rather than all landing together. - let mut counts: HashMap = HashMap::new(); - for b in 0..=255u8 { - let w = migration_wave_for(Some(&peer_id(b)), 7); - assert!(w < waves, "wave {w} outside 0..{waves}"); - *counts.entry(w).or_default() += 1; - } - assert_eq!( - counts.len() as u64, - waves, - "every wave should be occupied, got {counts:?}" - ); - } - - #[test] - fn waves_are_actually_staggered_under_the_shipped_defaults() { - // The combination is what matters, not either setting alone. Measured from first - // start, a 72 hour hold and 24 hour waves cancel out: waves would open at 0, 24, - // 48 and 72 hours while nothing may shed until 72, so every wave is open the - // moment the first one can act and the whole close group moves together. Measured - // from the end of the hold, they stagger as intended. - let config = MigrationConfig::default(); - assert_eq!(config.shed_hold_hours, 72); - assert_eq!(config.wave_hours, 24); - - let mut state = MigrationState::new(MigrationPhase::Bridging); - let waves = migration_wave_count(7); - assert_eq!(waves, 4); - - // Nothing is open before the hold ends. - state.first_start_unix = now_unix(); - for w in 0..waves { - assert!( - !wave_has_opened(&state, &config, w), - "wave {w} opened too early" - ); - } - - // At the end of the hold, exactly the first wave is open. - state.first_start_unix = now_unix().saturating_sub(72 * 3600 + 60); - assert!(wave_has_opened(&state, &config, 0)); - for w in 1..waves { - assert!( - !wave_has_opened(&state, &config, w), - "wave {w} must wait its turn, or the group migrates together" - ); - } - - // Each later wave opens one wave_hours after the one before it. - for open in 1..waves { - state.first_start_unix = now_unix().saturating_sub((72 + open * 24) * 3600 + 60); - for w in 0..=open { - assert!(wave_has_opened(&state, &config, w)); - } - for w in open + 1..waves { - assert!(!wave_has_opened(&state, &config, w)); - } - } - } - - #[test] - fn an_implausible_clock_restarts_the_holds_rather_than_voiding_them() { - let dir = TempDir::new().expect("temp dir"); - let config = MigrationConfig::default(); - - // Zero is what a node with an unsynchronised clock writes at first boot, and it - // would make every hold vacuous. So would a time in the future. - let mut state = MigrationState::new(MigrationPhase::Committed); - state.first_start_unix = 0; - state.committed_at_unix = Some(0); - state.save(dir.path()).expect("save"); - - let loaded = MigrationState::load_or_new(dir.path(), MigrationPhase::Bridging); - assert!( - !loaded.shed_hold_elapsed(&config), - "the hold must not be void" - ); - assert!(!loaded.retire_delay_elapsed(&config)); - - let mut future = MigrationState::new(MigrationPhase::Committed); - future.first_start_unix = now_unix().saturating_add(10 * 365 * 24 * 3600); - future.committed_at_unix = Some(future.first_start_unix); - future.save(dir.path()).expect("save"); - let loaded = MigrationState::load_or_new(dir.path(), MigrationPhase::Bridging); - assert!(!loaded.shed_hold_elapsed(&config)); - assert!(!loaded.retire_delay_elapsed(&config)); - } - - #[test] - fn copy_reports_accumulate_across_passes() { - let mut total = CopyReport::default(); - total.merge(CopyReport { - copied: 3, - bytes: 300, - ..CopyReport::default() - }); - total.merge(CopyReport { - copied: 2, - bytes: 200, - unusable: 1, - stopped_for_space: true, - ..CopyReport::default() - }); - assert_eq!(total.copied, 5); - assert_eq!(total.bytes, 500); - assert_eq!(total.unusable, 1); - assert!(total.stopped_for_space); - } - /// A peer that received the commitment and then left the group is not evidence. - /// - /// It is not going to audit this node, so counting it lets a node give chunks up - /// while the neighbours who will audit it still hold it to the old, larger key set. - #[test] - fn a_departed_peer_that_knows_the_commitment_does_not_open_the_gate() { - let received: HashSet = (0..6).map(peer_id).collect(); - let still_here: Vec = (0..3).map(peer_id).collect(); - let joined_since: Vec = (100..103).map(peer_id).collect(); - let current: Vec = still_here - .iter() - .chain(joined_since.iter()) - .copied() - .collect(); - - // Six peers know it and the group is six wide, so a count that ignores who is - // actually here would sail past the threshold. - assert_eq!(received.len(), 6); - assert_eq!(current.len(), 6); - assert!(!enough_of_the_group_knows(&received, ¤t, 5)); - - // Only the three that are both here and informed count. - assert!(enough_of_the_group_knows(&received, ¤t, 3)); - assert!(!enough_of_the_group_knows(&received, ¤t, 4)); - } - - #[test] - fn a_group_that_has_all_seen_the_commitment_opens_the_gate() { - let group: Vec = (0..6).map(peer_id).collect(); - let received: HashSet = group.iter().copied().collect(); - assert!(enough_of_the_group_knows(&received, &group, 5)); - } - - #[test] - fn no_peer_ever_satisfies_a_zero_threshold() { - let group: Vec = (0..6).map(peer_id).collect(); - let received: HashSet = group.iter().copied().collect(); - // A group this node cannot reason about must not be read as unanimous consent. - assert!(!enough_of_the_group_knows(&received, &group, 0)); - } - - /// The gate stays shut when routing cannot show a full close group at all. - /// - /// Without a routing view there is no way to tell an informed neighbour from a - /// departed one, and an unanswerable question must not read as a yes. - #[tokio::test] - async fn without_a_routing_view_the_commitment_gate_stays_shut() { - let context = MigrationContext { - p2p: None, - self_id: None, - self_xor: None, - commitment: None, - replication: None, - sync_state: None, - audit_challenge_coordinator: None, - peer_commitments: None, - close_group_size: 7, - }; - assert!(!context.neighbours_know_the_commitment().await); - } -} diff --git a/src/storage/migration_signal.rs b/src/storage/migration_signal.rs new file mode 100644 index 00000000..287f2662 --- /dev/null +++ b/src/storage/migration_signal.rs @@ -0,0 +1,753 @@ +//! What a node tells the network about its move off the old chunk store. +//! +//! The release that finally deletes the old store has to be published at a moment when the +//! fleet has finished moving, and "the fleet has finished" is not something a calendar can +//! establish. Nor can our own logs: they cover the nodes we run, and the nodes most likely +//! to still be carrying a `chunks.mdb` are the ones we do not. +//! +//! So a node says so itself, in the one field every peer already sees. `saorsa-core` sends a +//! user agent string with every signed message and keeps each peer's, so any node can ask +//! what its neighbours are. Putting the answer there costs no new message, no new field and +//! no protocol version: it is a different value in a string that was already on the wire. +//! +//! Read [`report_until_shutdown`] before using any of this to decide a release. It does not +//! establish that the fleet has finished, and cannot. A node sees only the peers it is +//! connected to, and each of those answers as of its own last start, so the most this can show +//! is that some peer reported an old store when it last started. It can never show that no node +//! has one. What it gives is the only view we get of the nodes we do not run. +//! +//! Two rules the string has to obey. It must still begin `node/`, because that prefix is +//! what `saorsa-core` uses to decide whether a peer is a DHT participant at all, and a node +//! that loses it stops being routed to. And the three states must never be folded into two: +//! a directory this node could not read is not the same as one that is not there, and +//! reading "cannot tell" as "finished" is how a gate comes back clean over a fleet that is +//! not. +//! +//! What is deliberately NOT here: anything about whether storage is switched off. A node +//! with `storage.enabled = false` never opens a store, but the old environment is still on +//! its disk and the release that deletes it will still find it. The question this answers is +//! about the filesystem, so it is asked of the filesystem, whatever the node was configured +//! to do with it. + +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Weak}; +use std::time::Duration; + +use saorsa_core::P2PNode; +use tokio_util::sync::CancellationToken; + +use crate::logging::{info, warn}; + +/// How often a node says where it is and what it can see. +/// +/// Often enough that a node's reading **of its own disk** is never many hours stale, rarely +/// enough that it is a line an operator can read rather than a stream. What it says about its +/// peers is not fresh at any cadence: their user agents were fixed when their transports were +/// built, so a peer's answer is as of its last start whenever this runs. +/// +/// It is a heartbeat as much as a count: a node that stops saying anything is a node the +/// release gate must treat as unfinished, and it can only do that if a healthy node says +/// something on a known cadence. +const REPORT_INTERVAL: Duration = Duration::from_secs(15 * 60); + +/// The directory the old chunk store lives in. +pub const LEGACY_ENV_DIR: &str = "chunks.mdb"; + +/// What retirement renames it to before deleting it. +pub(super) const RETIRED_SUFFIX: &str = ".retired"; + +/// The file retirement writes inside a directory to say it has finished with it. +pub(super) const RETIRED_MARKER: &str = "RETIRED"; + +/// The token that carries the state, so a reader can find it wherever it sits. +const SIGNAL_PREFIX: &str = "migration/"; + +/// Where this node is in the move off the old chunk store, as seen from its own disk. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum MigrationSignal { + /// Something is still there that this node has not finished with. + Legacy, + /// Nothing is, or only the harmless remains of a cleanup that did not quite finish. + Files, + /// The disk could not be read well enough to say. Never folded into either answer. + Unknown, +} + +impl MigrationSignal { + /// The token this state appears as on the wire. + const fn token(self) -> &'static str { + match self { + Self::Legacy => "legacy", + Self::Files => "files", + Self::Unknown => "unknown", + } + } + + /// Read the state off this node's own disk. + /// + /// Cheap enough to call before the transport is built, which is where it has to be + /// called: the user agent is fixed when the transport is constructed. + #[must_use] + pub fn from_disk(root_dir: &Path) -> Self { + let Ok(dirs) = legacy_directories(root_dir) else { + return Self::Unknown; + }; + let mut answer = Self::Files; + for dir in dirs { + match classify(&dir) { + // Finished with, or empty, which is what an interrupted cleanup leaves. + // Neither holds a chunk, so neither makes this node unfinished. + Leftover::Harmless => {} + Leftover::Holding => return Self::Legacy, + // Keep looking: a directory further down the list may still be holding + // chunks, and that is the stronger answer of the two. + Leftover::Unreadable => answer = Self::Unknown, + } + } + answer + } +} + +/// What one leftover directory means for the node carrying it. +/// +/// `legacy_artifacts` decides what it may delete from this same verdict, so that the state a +/// node reports and the state its cleanup acts on can never be two different readings of one +/// directory. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Leftover { + /// It holds chunks this node has not moved. + Holding, + /// It holds nothing, or it carries the mark that says it was finished with. + Harmless, + /// It could not be read well enough to say which. + Unreadable, +} + +/// Every leftover of the old chunk store under `root_dir`, live name and tombstones alike. +/// +/// The names are matched exactly rather than by prefix. Retirement only ever creates +/// `chunks.mdb.retired` or `chunks.mdb.retired.`, and a prefix match would also claim a +/// directory somebody else put there, which matters because a later release deletes what +/// this list returns. +/// +/// An entry that cannot be read is returned rather than skipped, so it becomes `Unknown` +/// rather than silently becoming `Files`. +pub(super) fn legacy_directories(root_dir: &Path) -> Result, Unreadable> { + let mut found = Vec::new(); + + // `symlink_metadata`, not `try_exists`: the latter follows links, so a dangling or + // looping one at the live name would read as nothing being there. + let live = root_dir.join(LEGACY_ENV_DIR); + // An error is not an absence: a live name that cannot be queried hides an environment + // that may well be there, so it goes on the list and becomes `Unknown` rather than + // quietly becoming `Files`. + if !matches!(std::fs::symlink_metadata(&live), Err(ref e) if e.kind() == std::io::ErrorKind::NotFound) + { + found.push(live); + } + + let entries = match std::fs::read_dir(root_dir) { + Ok(entries) => entries, + // A root that is not there yet holds nothing, which is every node starting for the + // first time. That is an answer. + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(found), + // One that cannot be listed hides every tombstone in it, so there is no answer to + // give. Saying so is the whole reason `Unknown` exists. + Err(_) => return Err(Unreadable), + }; + for entry in entries { + // Nor is one unreadable entry evidence that there is nothing behind it. An earlier + // version of this pushed a made-up path here so the caller would classify it, and a + // made-up path that happens not to exist classifies as harmless: one unreadable + // directory entry could hide a real tombstone and still produce `files`, which is + // exactly the false green a release gate must not be able to show. + let Ok(entry) = entry else { + return Err(Unreadable); + }; + if entry.file_name().to_str().is_some_and(is_tombstone_name) { + found.push(entry.path()); + } + } + Ok(found) +} + +/// There is no answer to give about this root. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) struct Unreadable; + +/// The most tombstones one root can hold, matching what retirement will ever create. +pub(super) const MAX_TOMBSTONES: u32 = 64; + +/// Is this a name retirement gives a tombstone? +/// +/// `chunks.mdb.retired`, or that plus `.` for `n` in `1..=64`, written the way retirement +/// writes it. The bounds are not decoration: retirement only ever counts up to 64, so `.65` +/// and `.007` are names it cannot have produced, and this list becomes a list of directories +/// a later release deletes. +pub(super) fn is_tombstone_name(name: &str) -> bool { + let base = format!("{LEGACY_ENV_DIR}{RETIRED_SUFFIX}"); + if name == base { + return true; + } + let Some(suffix) = name.strip_prefix(&format!("{base}.")) else { + return false; + }; + // Parsed and then written back out, so a leading zero or a plus sign fails to match + // itself: `"007".parse::()` is happily 7, and `chunks.mdb.retired.007` is not a + // name anything here created. + suffix + .parse::() + .is_ok_and(|n| (1..=MAX_TOMBSTONES).contains(&n) && suffix == n.to_string()) +} + +/// What one directory says about itself. +pub(super) fn classify(dir: &Path) -> Leftover { + match std::fs::symlink_metadata(dir) { + // A link is never treated as finished with, whatever it points at: the mark would + // have been written through it into a directory that is not this node's. It is also + // never followed to see what is behind it. + Ok(meta) if meta.file_type().is_symlink() => return Leftover::Holding, + Ok(_) => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Leftover::Harmless, + Err(_) => return Leftover::Unreadable, + } + // A regular file, not merely something at that name. Retirement writes the mark with + // `create_new`, so it is always an ordinary file; a directory, a link, a FIFO or anything + // else wearing the name is not evidence of anything, and this answer is what decides + // whether a later release deletes the chunks underneath it. + match std::fs::symlink_metadata(dir.join(RETIRED_MARKER)) { + Ok(meta) if meta.is_file() => return Leftover::Harmless, + Ok(_) => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(_) => return Leftover::Unreadable, + } + // No mark. A directory with nothing in it holds no chunks, so it cannot be hiding any: + // that is what a cleanup interrupted between emptying a tombstone and removing it + // leaves behind. + std::fs::read_dir(dir).map_or(Leftover::Unreadable, |mut entries| { + if entries.next().is_none() { + Leftover::Harmless + } else { + Leftover::Holding + } + }) +} + +/// The user agent this node announces itself with. +/// +/// Keeps the `node/` prefix `saorsa-core` gates DHT membership on, reports this build's +/// version rather than the transport's, because that is the one a release decision is made +/// about, and carries the migration state as its own token. +#[must_use] +pub fn user_agent(signal: MigrationSignal) -> String { + format!( + "node/{} {SIGNAL_PREFIX}{}", + env!("CARGO_PKG_VERSION"), + signal.token() + ) +} + +/// What a peer's user agent says about that peer. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PeerMigrationState { + /// It says it still has an old chunk store. + Legacy, + /// It says it has finished. + Files, + /// It says it cannot tell. + Unknown, + /// It says nothing, so it is running a build from before this was reported. Counted on + /// its own rather than with the finished ones: silence is not completion. + Unreported, + /// Not a node at all. Clients connect and announce themselves too, and counting them as + /// nodes that never reported would make every reading look worse than it is. + NotANode, +} + +/// Read a peer's user agent. +#[must_use] +pub fn peer_state(user_agent: &str) -> PeerMigrationState { + if !user_agent.starts_with("node/") { + return PeerMigrationState::NotANode; + } + for token in user_agent.split_whitespace() { + let Some(state) = token.strip_prefix(SIGNAL_PREFIX) else { + continue; + }; + return match state { + "legacy" => PeerMigrationState::Legacy, + "files" => PeerMigrationState::Files, + // A token we do not recognise is a build that reports something this one has + // never heard of. That is not "finished". + _ => PeerMigrationState::Unknown, + }; + } + PeerMigrationState::Unreported +} + +/// One tally of what a node can see around it. +/// +/// Every field counts what a peer **announced**, which it fixed at its last start. None of them +/// says what that peer holds now. And an all-zero tally is not an answer: a node connected to +/// nobody produces one, and it reads exactly like a tally of peers that have all finished. The +/// number of peers seen is what tells those apart, which is why it is on the line. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct PeerTally { + /// Peers that announced an old chunk store at their last start. + pub legacy: usize, + /// Peers that announced having finished, as of their last start. + pub files: usize, + /// Peers that announced they could not tell, or answered with something this build does + /// not know. + pub unknown: usize, + /// Peers running a build from before this was reported. + pub unreported: usize, +} + +impl PeerTally { + /// Peers that are not evidence the fleet has finished. + #[must_use] + pub const fn outstanding(self) -> usize { + self.legacy + self.unknown + self.unreported + } + + fn add(&mut self, state: PeerMigrationState) { + match state { + PeerMigrationState::Legacy => self.legacy += 1, + PeerMigrationState::Files => self.files += 1, + PeerMigrationState::Unknown => self.unknown += 1, + PeerMigrationState::Unreported => self.unreported += 1, + // Deliberately not counted at all. A client is not a node that failed to + // report, and putting it in any of the buckets above would make every reading + // worse than it is. + PeerMigrationState::NotANode => {} + } + } +} + +/// Count what this node can see of its neighbours. +/// +/// These are edges, not nodes: two of our nodes connected to the same peer both report it, +/// and a peer nobody is connected to is in nobody's count. That is why each line carries the +/// observer, so whoever adds them up can decide what a peer is worth rather than trusting an +/// arithmetic sum. +pub async fn tally_peers(p2p: &Arc) -> PeerTally { + let mut tally = PeerTally::default(); + let transport = p2p.transport(); + let observer = p2p.peer_id().to_hex(); + for peer in transport.connected_peers().await { + // No agent recorded is not the same as a peer that reported nothing, but it is + // just as far from evidence of completion, so it lands in the same bucket rather + // than being skipped. + let agent = transport.peer_user_agent(&peer).await; + let state = agent + .as_deref() + .map_or(PeerMigrationState::Unreported, peer_state); + // One line per peer, not just the totals. What a node sees are edges: two of our + // nodes connected to the same peer both report it, and a peer nobody is connected to + // is in nobody's count. Summing the totals across the fleet therefore counts some + // nodes twice and others never, which is not a number a release decision can rest + // on. With the observer, the peer and the moment on each line, whoever adds them up + // can deduplicate by peer and apply their own freshness rule; without them, they + // cannot. + // + // At `info`, not `debug`. Nodes run at `info` (`cli.rs:95`), so the same line at + // `debug` is written nowhere the gate can read it, and the release would be decided + // on the aggregates alone — which is the number that cannot be deduplicated. A line + // nobody emits is not a signal. + // + // Only for peers that are not reporting finished. The gate asks which distinct nodes + // are still outstanding, so those are the ones that need naming, and the cost falls + // away as they stop being outstanding rather than peaking when they do. Note which way + // that runs: these lines stopping means no peer this node is connected to is still + // reporting an old store, which is not the same as the fleet having finished, and + // never can be. The aggregate line below is emitted either way and + // carries the finished count, so the denominator does not go missing with them, and + // a node that has gone quiet is still distinguishable from a node with nothing to + // report. + if state != PeerMigrationState::Files && state != PeerMigrationState::NotANode { + info!( + migration_event = "peer_state", + observer = %observer, + peer = %peer.to_hex(), + state = peer_state_token(state), + agent = agent.as_deref().unwrap_or("none"), + "Storage migration: peer {} is {}", + peer.to_hex(), + peer_state_token(state) + ); + } + tally.add(state); + } + tally +} + +/// The token a peer's state is reported as, so the aggregate line and the per-peer lines +/// cannot drift apart. +/// +/// Only ever read by a log line, so it goes when the logging feature does. +#[cfg_attr(not(feature = "logging"), allow(dead_code))] +const fn peer_state_token(state: PeerMigrationState) -> &'static str { + match state { + PeerMigrationState::Legacy => "legacy", + PeerMigrationState::Files => "files", + PeerMigrationState::Unknown => "unknown", + PeerMigrationState::Unreported => "unreported", + PeerMigrationState::NotANode => "not-a-node", + } +} + +/// Say where this node is, and what it can see of its neighbours, until it shuts down. +/// +/// Two questions, and they are not the same one. **This node's own state** is read from its +/// disk every pass, so a node that finishes says so within the interval rather than at its next +/// restart. **What it sees of its peers** is read from their user agents. +/// +/// What the peer half means, in the fewest words that are all true, because a release decision +/// rests on it. +/// +/// It counts what the peers this node is connected to **announced**, each as of that peer's own +/// last start. `saorsa-core` copies the user agent when it builds the transport, so a node that +/// finishes migrating goes on announcing `legacy` until it restarts. +/// +/// Two consequences, and they run in opposite directions, so the tally bounds nothing. A peer +/// announcing `legacy` may have finished since, so the count can be too high. A node that is +/// offline, or simply not connected to, is absent from it, so the count can be too low. An +/// all-zero tally proves nothing on its own either, because a node connected to nobody produces +/// one; the number of peers seen is what tells that apart, which is why it is on the line. +/// +/// So this can surface nodes that have not finished. It cannot establish that none remain, and +/// no amount of it adds up to that. `outstanding` counts `legacy`, `unknown` and `unreported` +/// together, because a peer whose disk could not be read and a peer on a build from before this +/// existed are both as far from finished as `legacy` is. +/// +/// It is still the only way our own fleet learns anything at all about the nodes we do not run. +/// +/// The handle is **weak**. A reporter must never be the reason the thing it observes stays +/// alive: a strong one would keep a dropped node's transport, and its bound port, for as long +/// as this task ran. +pub async fn report_until_shutdown( + p2p: Weak, + root_dir: std::path::PathBuf, + shutdown: CancellationToken, +) { + loop { + // Wait first. A node that has just started has no peers to describe, and nothing has + // changed on its disk since the user agent was built from it. + tokio::select! { + () = shutdown.cancelled() => return, + () = tokio::time::sleep(REPORT_INTERVAL) => {} + } + let Some(node) = p2p.upgrade() else { + return; + }; + let own = MigrationSignal::from_disk(&root_dir); + let peers = tally_peers(&node).await; + drop(node); + let seen = peers.outstanding() + peers.files; + if own == MigrationSignal::Legacy || own == MigrationSignal::Unknown { + warn!( + migration_event = "signal", + state = own.token(), + peers_legacy = peers.legacy, + peers_unknown = peers.unknown, + peers_unreported = peers.unreported, + peers_files = peers.files, + "This node cannot report itself finished with the old chunk store ({}: \ + `legacy` means one is there, `unknown` means its disk could not be read, so \ + whether one is there is not known). {} of the {seen} node(s) it can see are \ + not reporting finished either.", + own.token(), + peers.outstanding() + ); + } else { + info!( + migration_event = "signal", + state = own.token(), + peers_legacy = peers.legacy, + peers_unknown = peers.unknown, + peers_unreported = peers.unreported, + peers_files = peers.files, + "Storage migration: this node has nothing of the old store left; {} of the \ + {seen} node(s) it can see are not reporting finished.", + peers.outstanding() + ); + } + } +} + +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::expect_used)] +mod tests { + + /// The reporter must not keep the node it reports on alive. + /// + /// It holds a weak handle and upgrades per pass. A strong one would keep the node, and + /// with it the transport and the bound port, for as long as the task ran, so any path + /// that dropped a node without cancelling its token would leak a live port instead of + /// stopping a node. Nothing notices that until the next bind fails, somewhere else, + /// much later. + #[tokio::test] + async fn the_reporter_lets_go_of_a_node_that_was_dropped() { + let dir = tempfile::tempdir().expect("temp dir"); + let root = dir.path().to_path_buf(); + let shutdown = CancellationToken::new(); + + // Stand in for the node: what matters is that the task holds no strong reference, + // so the count of strong holders does not rise when the reporter starts, and the + // reporter stops on its own once the last real holder goes. + let owner = Arc::new(()); + let weak = Arc::downgrade(&owner); + assert_eq!(Arc::strong_count(&owner), 1); + + let handle = tokio::spawn({ + let weak = weak.clone(); + let shutdown = shutdown.clone(); + async move { + loop { + tokio::select! { + () = shutdown.cancelled() => return "cancelled", + () = tokio::time::sleep(std::time::Duration::from_millis(5)) => {} + } + let Some(up) = weak.upgrade() else { + return "node went away"; + }; + drop(up); + } + } + }); + + assert_eq!( + Arc::strong_count(&owner), + 1, + "starting the reporter must not add a strong holder" + ); + drop(owner); + let outcome = tokio::time::timeout(std::time::Duration::from_secs(5), handle) + .await + .expect("the reporter must stop on its own") + .expect("task must not panic"); + assert_eq!( + outcome, "node went away", + "the reporter must stop when the node is gone, not wait for a cancellation \ + nobody sends" + ); + let _ = root; + } + + use super::*; + use tempfile::TempDir; + + fn dir_with(root: &Path, name: &str) -> std::path::PathBuf { + let path = root.join(name); + std::fs::create_dir_all(&path).unwrap(); + path + } + + #[test] + fn a_node_with_nothing_on_disk_has_finished() { + let root = TempDir::new().unwrap(); + assert_eq!( + MigrationSignal::from_disk(root.path()), + MigrationSignal::Files + ); + } + + #[test] + fn a_root_that_does_not_exist_yet_has_finished() { + let root = TempDir::new().unwrap(); + let never = root.path().join("not-created"); + assert_eq!(MigrationSignal::from_disk(&never), MigrationSignal::Files); + } + + #[test] + fn a_live_environment_with_chunks_in_it_has_not() { + let root = TempDir::new().unwrap(); + let env = dir_with(root.path(), LEGACY_ENV_DIR); + std::fs::write(env.join("data.mdb"), b"chunks").unwrap(); + assert_eq!( + MigrationSignal::from_disk(root.path()), + MigrationSignal::Legacy + ); + } + + #[test] + fn a_marked_leftover_has_finished() { + let root = TempDir::new().unwrap(); + let env = dir_with(root.path(), LEGACY_ENV_DIR); + std::fs::write(env.join("data.mdb"), b"chunks").unwrap(); + std::fs::write(env.join(RETIRED_MARKER), b"").unwrap(); + assert_eq!( + MigrationSignal::from_disk(root.path()), + MigrationSignal::Files + ); + } + + #[test] + fn an_empty_leftover_has_finished() { + // What a cleanup interrupted between emptying a tombstone and removing it leaves. + let root = TempDir::new().unwrap(); + dir_with(root.path(), "chunks.mdb.retired"); + assert_eq!( + MigrationSignal::from_disk(root.path()), + MigrationSignal::Files + ); + } + + #[test] + fn a_tombstone_with_chunks_in_it_has_not() { + // A crash between the rename and the mark leaves an intact environment wearing a + // retired-looking name. What it is called is not evidence. + let root = TempDir::new().unwrap(); + let tomb = dir_with(root.path(), "chunks.mdb.retired.3"); + std::fs::write(tomb.join("data.mdb"), b"chunks").unwrap(); + assert_eq!( + MigrationSignal::from_disk(root.path()), + MigrationSignal::Legacy + ); + } + + #[test] + fn an_entry_that_cannot_be_read_is_never_read_as_finished() { + // An earlier version pushed a made-up path when an entry could not be read, so the + // caller would classify it. A made-up path that happens not to exist classifies as + // harmless, so one unreadable entry could hide a real tombstone and still answer + // `files`. A gate that can come back green over a fleet that has not finished is + // worse than no gate. + let root = TempDir::new().unwrap(); + let unreadable = root.path().join("locked"); + std::fs::create_dir_all(&unreadable).unwrap(); + let tomb = unreadable.join("chunks.mdb.retired"); + std::fs::create_dir_all(&tomb).unwrap(); + std::fs::write(tomb.join("data.mdb"), b"chunks").unwrap(); + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&unreadable, std::fs::Permissions::from_mode(0o000)).unwrap(); + let answer = MigrationSignal::from_disk(&unreadable); + std::fs::set_permissions(&unreadable, std::fs::Permissions::from_mode(0o755)).unwrap(); + assert_eq!( + answer, + MigrationSignal::Unknown, + "a root that cannot be listed must never answer that it has finished" + ); + } + } + + #[test] + fn a_directory_that_only_looks_like_a_tombstone_is_not_one() { + // This list is eventually a list of directories a later release deletes, so it + // matches the names retirement actually creates and nothing else. + assert!(is_tombstone_name("chunks.mdb.retired")); + assert!(is_tombstone_name("chunks.mdb.retired.1")); + assert!(is_tombstone_name("chunks.mdb.retired.42")); + assert!(!is_tombstone_name("chunks.mdb.retired-mine")); + assert!(!is_tombstone_name("chunks.mdb.retired.")); + assert!(!is_tombstone_name("chunks.mdb.retired.backup")); + assert!(!is_tombstone_name("chunks.mdb")); + // Names retirement counts up to, and names it never reaches. `.007` parses as 7 and + // is still not a name anything wrote. + assert!(is_tombstone_name("chunks.mdb.retired.64")); + assert!(!is_tombstone_name("chunks.mdb.retired.65")); + assert!(!is_tombstone_name("chunks.mdb.retired.0")); + assert!(!is_tombstone_name("chunks.mdb.retired.007")); + assert!(!is_tombstone_name("chunks.mdb.retired.+1")); + assert!(!is_tombstone_name("chunks.mdb.retired.999999")); + + let root = TempDir::new().unwrap(); + let mine = dir_with(root.path(), "chunks.mdb.retired-mine"); + std::fs::write(mine.join("data.mdb"), b"somebody else's").unwrap(); + assert_eq!( + MigrationSignal::from_disk(root.path()), + MigrationSignal::Files + ); + } + + #[test] + fn a_linked_environment_is_never_read_as_finished() { + // The mark would have been written through the link into a directory this node + // does not own, so a link is never evidence that anything was finished with, and + // what it points at is never followed. + let root = TempDir::new().unwrap(); + let elsewhere = dir_with(root.path(), "elsewhere"); + std::fs::write(elsewhere.join(RETIRED_MARKER), b"").unwrap(); + #[cfg(unix)] + { + std::os::unix::fs::symlink(&elsewhere, root.path().join(LEGACY_ENV_DIR)).unwrap(); + assert_eq!( + MigrationSignal::from_disk(root.path()), + MigrationSignal::Legacy + ); + } + } + + /// A node still running the release before this one reads as unreported, never as done. + /// + /// This release lands on a fleet where most nodes are still on the previous one, and + /// those announce no migration token at all. Counting them as finished would let the gate + /// come back clean over a fleet that has barely started. They get their own bucket, and + /// `outstanding` includes it. + #[test] + fn a_peer_on_the_previous_release_is_not_counted_as_finished() { + assert_eq!( + peer_state("node/0.19.0"), + PeerMigrationState::Unreported, + "a node with no migration token has not reported, which is not the same as done" + ); + let mut tally = PeerTally::default(); + tally.add(peer_state("node/0.19.0")); + assert_eq!(tally.files, 0, "it must not land in the finished bucket"); + assert_eq!(tally.outstanding(), 1, "and must count against readiness"); + } + + #[test] + fn the_user_agent_keeps_the_prefix_that_gates_dht_membership() { + // saorsa-core decides whether a peer is a DHT participant by this prefix alone. A + // node that loses it stops being routed to, which is a much worse outcome than not + // reporting at all, so it is worth pinning. + for signal in [ + MigrationSignal::Legacy, + MigrationSignal::Files, + MigrationSignal::Unknown, + ] { + assert!(user_agent(signal).starts_with("node/")); + } + } + + #[test] + fn a_peer_reads_back_what_a_node_announced() { + assert_eq!( + peer_state(&user_agent(MigrationSignal::Legacy)), + PeerMigrationState::Legacy + ); + assert_eq!( + peer_state(&user_agent(MigrationSignal::Files)), + PeerMigrationState::Files + ); + assert_eq!( + peer_state(&user_agent(MigrationSignal::Unknown)), + PeerMigrationState::Unknown + ); + } + + #[test] + fn silence_is_counted_as_silence_and_not_as_completion() { + // The build before this one announces the transport's own agent, with no token of + // ours. Reading that as "finished" is exactly how a gate comes back clean over a + // fleet that has not finished. + assert_eq!(peer_state("node/0.27.0"), PeerMigrationState::Unreported); + assert_eq!( + peer_state("node/0.17.2 migration/something-new"), + PeerMigrationState::Unknown + ); + } + + #[test] + fn a_client_is_not_a_node_that_failed_to_report() { + // Clients authenticate and announce themselves too. Counting them among the peers + // that never reported would make every reading look worse than it is, and the + // count is what a release decision is made on. + assert_eq!(peer_state("client/0.27.0"), PeerMigrationState::NotANode); + } +} diff --git a/src/storage/mod.rs b/src/storage/mod.rs index bda34ac8..49640827 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -1,8 +1,8 @@ //! Storage subsystem for chunk persistence. //! -//! This module provides content-addressed LMDB storage for chunks, -//! along with a protocol handler that integrates with saorsa-core's -//! `Protocol` trait for automatic message routing. +//! This module provides content-addressed storage for chunks, one immutable file per +//! chunk, along with a protocol handler that integrates with saorsa-core's `Protocol` +//! trait for automatic message routing. //! //! # Architecture //! @@ -19,7 +19,7 @@ //! │ QuoteRequest ChunkPutRequest ChunkGetRequest //! │ │ │ │ │ //! │ ▼ ▼ ▼ │ -//! │ QuoteGenerator PaymentVerifier LmdbStorage│ +//! │ QuoteGenerator PaymentVerifier ChunkStore│ //! │ │ │ │ │ //! │ └─────────────────────────┴─────────────────┘ │ //! │ │ │ @@ -31,11 +31,11 @@ //! //! ```rust,ignore //! use std::sync::Arc; -//! use ant_node::storage::{AntProtocol, LmdbStorage, LmdbStorageConfig}; +//! use ant_node::storage::{AntProtocol, ChunkStore, ChunkStoreConfig}; //! //! // Create storage -//! let config = LmdbStorageConfig::default(); -//! let storage = Arc::new(LmdbStorage::new(config).await?); +//! let config = ChunkStoreConfig::default(); +//! let storage = Arc::new(ChunkStore::new(config).await?); //! //! // Create protocol handler //! let protocol = AntProtocol::new(storage, Arc::new(payment_verifier), Arc::new(quote_generator)); @@ -44,23 +44,32 @@ //! listener.register_protocol(protocol).await?; //! ``` -pub(crate) mod chunk_store; +// `test-utils` makes this module public so integration tests and downstream harnesses can +// reach the store directly. Anything `pub` inside it is therefore public in that build, and +// this list of re-exports below is the API boundary that actually holds — not the item +// visibilities inside the module. Adding a `pub` item there is not a decision to publish it; +// flipping this cfg would be. #[cfg(any(test, feature = "test-utils"))] -pub mod file_store; +pub mod chunk_store; #[cfg(not(any(test, feature = "test-utils")))] -pub(crate) mod file_store; +pub(crate) mod chunk_store; mod handler; -pub(crate) mod lmdb; -pub mod migration; +// Both are this crate's own business. The cleanup is called once, from the node builder, and +// the signal was `pub(crate)` in the release that added it; exporting either would publish a +// migration this release exists to finish. +pub(crate) mod legacy_artifacts; +// Carried forward from the release before this one. Without it a node on this release reads +// to its peers as one that never reported at all, and the fleet gate that authorised this +// release could never come back clean again. +pub(crate) mod migration_signal; pub use crate::ant_protocol::XorName; -pub use chunk_store::{ChunkStore, ChunkStoreConfig, VerifyReport, LEGACY_ENV_DIR}; -pub use file_store::{FileStore, FileStoreConfig, StoreLayout}; +pub use chunk_store::{ChunkStore, ChunkStoreConfig}; +// Crate-private, as it was before the two stores became one: `CapacityVerdict` was +// `pub(crate)` on the old store and has no caller outside this crate. +pub(crate) use chunk_store::CapacityVerdict; pub use handler::AntProtocol; pub(crate) use handler::ChunkRequestContext; -pub(crate) use lmdb::CapacityVerdict; -pub use lmdb::{LmdbStorage, LmdbStorageConfig}; -pub use migration::{MigrationConfig, MigrationPhase, MigrationState}; /// Bytes in one MiB. pub const MIB: u64 = 1024 * 1024; diff --git a/tests/chunk_store_crash_safety.rs b/tests/chunk_store_crash_safety.rs new file mode 100644 index 00000000..1d90cd97 --- /dev/null +++ b/tests/chunk_store_crash_safety.rs @@ -0,0 +1,250 @@ +//! What survives a process dying part-way through a write. +//! +//! The store rests on being able to stop at any moment and start again: every step is +//! idempotent and re-derived from the filesystem. That is easy to assert and hard to +//! believe without trying it, so these tests kill a real child process at a real point in +//! the work and then open the store in this one and check what is there. +//! +//! **What this does and does not prove.** A killed process loses nothing the kernel has +//! already accepted, so this covers ordering and bookkeeping: a chunk is whole or absent +//! and never half-indexed, and what an interrupted write leaves behind is swept. It does +//! not cover power loss, where the kernel loses what it accepted and never wrote. That is +//! still a fleet gate. +//! +//! These tests came from the harness that proved the migration off the old chunk store. +//! Most of that harness went with the migration; these two did not belong to it. They are +//! about the store's own publish path, which is now the only one there is. + +#![allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + clippy::missing_panics_doc, + clippy::cast_possible_truncation +)] + +use ant_node::storage::{ChunkStore, ChunkStoreConfig}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::time::Duration; +use tempfile::TempDir; + +/// Run this test binary again as a child in the mode named by `role`, wait until it has +/// reached the named point, and kill it there. +/// +/// A child process rather than a thread, because the point is to lose everything the +/// process was holding: buffers, in-memory index, locks, half-finished intentions. +/// +/// The wait is a handshake, not a sleep. An earlier version of this slept and hoped, and +/// on a quick machine the child had finished everything before the kill arrived, so the +/// test was checking a clean shutdown while claiming to check a crash. The child now stops +/// at a failpoint inside the write and says so by writing a marker; this waits for the +/// marker and then kills it, so the process always dies at the same point in the same +/// operation. +fn kill_child_at_failpoint(role: &str, root: &Path, failpoint: &str, let_through: u64) -> PathBuf { + let marker = root.join(format!("reached-{role}")); + let _ = std::fs::remove_file(&marker); + + let exe = std::env::current_exe().expect("this test binary"); + let mut child = Command::new(exe) + .arg("--exact") + .arg(role) + .arg("--nocapture") + .arg("--ignored") + .env("ANT_CRASH_TEST_ROOT", root) + .env(failpoint, &marker) + .env( + ant_node::storage::chunk_store::HALT_AFTER, + let_through.to_string(), + ) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()) + .spawn() + .expect("spawn the child"); + + // Generous, but not unbounded. Without a deadline a failpoint that stopped working + // would hang the job rather than fail it, and a hang says nothing about the code. + let deadline = std::time::Instant::now() + Duration::from_secs(120); + while !marker.exists() { + if let Ok(Some(status)) = child.try_wait() { + panic!("the child exited before reaching the failpoint: {status}"); + } + if std::time::Instant::now() > deadline { + let _ = child.kill(); + panic!("the child never reached the failpoint"); + } + std::thread::sleep(Duration::from_millis(10)); + } + + child.kill().expect("kill the child"); + let _ = child.wait(); + marker +} + +/// Where the child was told to work. +fn child_root() -> PathBuf { + PathBuf::from(std::env::var("ANT_CRASH_TEST_ROOT").expect("the child needs a root")) +} + +/// Child mode: write chunks into a file store until killed. +#[tokio::test] +#[ignore = "child process of a crash test, not run on its own"] +async fn child_writes_until_killed() { + let root = child_root(); + let store = ChunkStore::new(ChunkStoreConfig { + root_dir: root, + disk_reserve: 0, + ..ChunkStoreConfig::default() + }) + .await + .expect("open"); + + // Always a chunk it has not written before, so the kill lands in real work rather + // than in a re-offer of something already on disk. An earlier version cycled the same + // hundred keys and spent almost all its time confirming duplicates. + let mut n = 0usize; + loop { + let content = chunk_bytes(n); + let address = ant_node::client::compute_address(&content); + let _ = store.put(&address, &content).await; + n += 1; + } +} + +/// A process killed inside a publish leaves no chunk it cannot serve. +/// +/// The child is stopped at the last moment before the chunk's name exists on disk: on Unix +/// the bytes written to a temporary file with the rename not yet made, off Unix the point +/// before the file is created at all, since that platform writes under the final name +/// because a rename there carries no durability guarantee. The failure this guards against +/// is the same on both: a name outliving its bytes. The index is built from filenames at +/// startup, so a partial file wearing a real chunk name would be advertised, committed to, +/// and unservable. +#[tokio::test] +async fn a_process_killed_mid_publish_leaves_no_chunk_it_cannot_serve() { + let tmp = TempDir::new().expect("temp dir"); + let root = tmp.path().join("node"); + std::fs::create_dir_all(&root).expect("mkdir"); + + // Twenty chunks land before the crash, so the store this reopens has real content in + // it. Stopping the very first write would leave nothing indexed and the loop below + // would pass by iterating over nothing. + let marker = kill_child_at_failpoint( + "child_writes_until_killed", + &root, + ant_node::storage::chunk_store::HALT_BEFORE_PUBLISH, + 20, + ); + assert!( + marker.exists(), + "the child must have reached the failpoint before it was killed" + ); + + // Reopening is itself part of the assertion: a store that cannot start after a crash + // is a node that cannot start. + let store = reopen(&root).await; + // The child discards its put results and the failpoint counts arrivals, not successes, + // so every publish before the kill could in principle have failed. An empty store makes + // the loop below pass over nothing, which is the one outcome that would let this test + // report success having checked no chunk at all. + let held = store.all_keys().await.expect("all_keys"); + assert!( + !held.is_empty(), + "the child published nothing before it was killed, so there is nothing to check" + ); + for key in held { + let served = store.get(&key).await; + assert!( + matches!(served, Ok(Some(_))), + "chunk {} is claimed after a crash but cannot be served: {served:?}", + hex::encode(key) + ); + } +} + +/// The temporary file a killed publish left behind is swept, not indexed. +/// +/// It carries no chunk name, so it can never be served, and leaving it would cost disk +/// for the life of the node. +/// +/// Unix only, because the leftover only exists on Unix. Off Unix the store creates the +/// file under its final name and flushes it, deliberately, since a rename there is not +/// documented to be durable. So there is no temporary file to sweep and the equivalent +/// hazard is different: a real chunk name over bytes that are short or wrong. That one is +/// covered by the store's own tests, which run on every platform: a read verifies the bytes +/// against the name and refuses a file that does not match. There used to be a second answer +/// here, the re-hash-everything pass retirement ran before deleting, and this release deletes +/// retirement, so it is no longer one. Forced power loss on a real filesystem remains an open +/// gate, named in ADR-0015. +#[cfg(unix)] +#[tokio::test] +async fn the_leftovers_of_a_killed_publish_are_swept() { + let tmp = TempDir::new().expect("temp dir"); + let root = tmp.path().join("node"); + std::fs::create_dir_all(&root).expect("mkdir"); + + kill_child_at_failpoint( + "child_writes_until_killed", + &root, + ant_node::storage::chunk_store::HALT_BEFORE_PUBLISH, + 5, + ); + + let before = temp_files(&root.join("chunks")); + assert!( + before > 0, + "the child should have left a temporary file behind when it was killed" + ); + + let store = reopen(&root).await; + store.wait_idle().await; + assert_eq!( + temp_files(&root.join("chunks")), + 0, + "the store should sweep what an interrupted write left" + ); + drop(store); +} + +/// How many partly-written files are under `chunks_dir`. +#[cfg(unix)] +fn temp_files(chunks_dir: &Path) -> usize { + let Ok(shards) = std::fs::read_dir(chunks_dir) else { + return 0; + }; + shards + .flatten() + .filter_map(|shard| std::fs::read_dir(shard.path()).ok()) + .flat_map(std::iter::IntoIterator::into_iter) + .flatten() + .filter(|entry| { + entry + .file_name() + .to_str() + .is_some_and(|name| !name.chars().all(|c| c.is_ascii_hexdigit())) + }) + .count() +} + +/// Open the store the way a restart would. +async fn reopen(root: &Path) -> ChunkStore { + let config = ChunkStoreConfig { + root_dir: root.to_path_buf(), + disk_reserve: 0, + ..ChunkStoreConfig::default() + }; + ChunkStore::new(config) + .await + .expect("the store must open after a crash") +} + +/// Deterministic content for chunk `n`. `n` goes in verbatim so no two differ only by a +/// wrap and collapse into one chunk. +fn chunk_bytes(n: usize) -> Vec { + let mut content = vec![0u8; 4096]; + content[..8].copy_from_slice(&(n as u64).to_le_bytes()); + for (i, byte) in content.iter_mut().enumerate().skip(8) { + *byte = ((i.wrapping_mul(17)).wrapping_add(n) % 251) as u8; + } + content +} diff --git a/tests/e2e/data_types/chunk.rs b/tests/e2e/data_types/chunk.rs index 2c875b76..650b8e08 100644 --- a/tests/e2e/data_types/chunk.rs +++ b/tests/e2e/data_types/chunk.rs @@ -358,7 +358,7 @@ mod tests { // so all Arc clones are released. // 2. Abort the protocol task that holds an Arc. // 3. Drop the node's own Arc. - // This ensures the LMDB env is fully closed before reopening. + // This ensures the chunk store is fully closed before reopening. let data_dir = { let node = harness .network_mut() diff --git a/tests/e2e/fresh_offer_capacity.rs b/tests/e2e/fresh_offer_capacity.rs index 5774c2a0..5b3fcecd 100644 --- a/tests/e2e/fresh_offer_capacity.rs +++ b/tests/e2e/fresh_offer_capacity.rs @@ -47,7 +47,7 @@ const UPLOAD_CHUNKS: usize = 48; /// one test process, so full-size chunks across every node would dominate the /// harness's memory. An admission slot is held per *offer* regardless of /// payload size, so the count above is what stresses the ceiling. The -/// trade-off is that the receiver's LMDB write is faster than production's, +/// trade-off is that the receiver's write is faster than production's, /// which is part of why this is a lower bound. const CHUNK_BYTES: usize = 64 * 1024; diff --git a/tests/e2e/replication.rs b/tests/e2e/replication.rs index abf5f92b..adc13dda 100644 --- a/tests/e2e/replication.rs +++ b/tests/e2e/replication.rs @@ -733,7 +733,7 @@ async fn test_paid_list_persistence() { dir }; - // Shut down the replication engine so the LMDB env is released + // Shut down the replication engine so the chunk store is released { let node = harness.network_mut().node_mut(3).expect("node"); if let Some(ref mut engine) = node.replication_engine { @@ -2916,7 +2916,7 @@ async fn scenario_43_paid_list_persists_across_restart() { dir }; - // Shut down the replication engine so the LMDB env is released + // Shut down the replication engine so the chunk store is released { let node = harness.network_mut().node_mut(3).expect("node"); if let Some(ref mut engine) = node.replication_engine { diff --git a/tests/e2e/subtree_audit_testnet.rs b/tests/e2e/subtree_audit_testnet.rs index bb0b5f70..53a0e1ab 100644 --- a/tests/e2e/subtree_audit_testnet.rs +++ b/tests/e2e/subtree_audit_testnet.rs @@ -3,7 +3,7 @@ //! //! These spin a real multi-node testnet and drive the SHIPPED audit over the //! live wire (real `handle_subtree_challenge` responder + `run_subtree_audit` -//! auditor + real LMDB storage), via the test-only `audit_peer_now` / +//! auditor + a real chunk store), via the test-only `audit_peer_now` / //! `rebuild_commitment_now` engine hooks. They prove the two outcomes that //! matter for a testnet: //! diff --git a/tests/e2e/testnet.rs b/tests/e2e/testnet.rs index 22995a3e..fdd7a725 100644 --- a/tests/e2e/testnet.rs +++ b/tests/e2e/testnet.rs @@ -11,7 +11,7 @@ //! - Message encoding/decoding (postcard serialization) //! - Content address verification //! - Payment verification (when enabled) -//! - LMDB storage persistence +//! - chunk store persistence use ant_node::ant_protocol::{ ChunkGetRequest, ChunkGetResponse, ChunkMessage, ChunkMessageBody, ChunkPutRequest, @@ -1037,7 +1037,7 @@ impl TestNetwork { /// Create a test node (but don't start it yet). /// /// Initializes the `AntProtocol` handler with: - /// - LMDB storage in the node's data directory + /// - the chunk store in the node's data directory /// - Payment verification configured per `TestNetworkConfig` /// - Quote generation with a test rewards address async fn create_node( @@ -1096,18 +1096,18 @@ impl TestNetwork { /// Create an `AntProtocol` handler for a test node. /// /// Configures: - /// - LMDB storage with verification enabled + /// - the chunk store with verification enabled /// - Payment verification (enabled/disabled based on `payment_enforcement`) /// - Quote generator with a test rewards address /// /// # Arguments /// - /// * `data_dir` - Directory for LMDB storage + /// * `data_dir` - Directory for the chunk store /// * `payment_enforcement` - Whether to enable EVM payment verification /// /// # Errors /// - /// Returns an error if LMDB storage initialisation fails. + /// Returns an error if the chunk store cannot be opened. pub async fn create_ant_protocol( data_dir: &std::path::Path, evm_network: Option, @@ -1120,14 +1120,14 @@ impl TestNetwork { /// /// # Errors /// - /// Returns an error if LMDB storage initialisation fails. + /// Returns an error if the chunk store cannot be opened. pub async fn create_ant_protocol_with_disk_reserve( data_dir: &std::path::Path, evm_network: Option, disk_reserve: u64, identity: &saorsa_core::identity::NodeIdentity, ) -> Result { - // Create LMDB storage + // Create the chunk store let storage_config = ChunkStoreConfig { root_dir: data_dir.to_path_buf(), disk_reserve, @@ -1135,7 +1135,7 @@ impl TestNetwork { }; let storage = ChunkStore::new(storage_config) .await - .map_err(|e| TestnetError::Core(format!("Failed to create LMDB storage: {e}")))?; + .map_err(|e| TestnetError::Core(format!("Failed to create the chunk store: {e}")))?; // Create payment verifier (EVM is always on). // When an EVM network is provided (e.g. Anvil), use it for on-chain verification. diff --git a/tests/migration_crash_safety.rs b/tests/migration_crash_safety.rs deleted file mode 100644 index 2bf464b2..00000000 --- a/tests/migration_crash_safety.rs +++ /dev/null @@ -1,542 +0,0 @@ -//! What survives a process dying part-way through the migration. -//! -//! The design rests on being able to stop at any moment and start again: every step is -//! idempotent and re-derived from the filesystem. That is easy to assert and hard to -//! believe without trying it, so these tests kill a real child process at a real point in -//! the work and then open the store in this one and check what is there. -//! -//! **What this does and does not prove.** A killed process loses nothing the kernel has -//! already accepted, so this covers ordering and bookkeeping: a chunk is whole or absent -//! and never half-indexed, what an interrupted write leaves behind is swept, and the store -//! always opens. It does not cover losing the page cache, which is what a real power cut -//! adds and what no hosted runner can do. That remains a fleet gate, and this is the part -//! of it that can be automated. -//! -//! The children stop at a named failpoint and say so, and the parent kills them there. An -//! earlier version slept and hoped; on a quick machine the child had finished before the -//! kill arrived, so the test was checking a clean shutdown while claiming to check a -//! crash. -//! -//! Runs on every platform CI covers, which is the filesystem matrix that matters: ext4, -//! APFS and NTFS. - -#![allow( - clippy::unwrap_used, - clippy::expect_used, - clippy::panic, - clippy::missing_panics_doc, - // Test fixtures: every cast here is of a bounded loop counter into a byte, and the - // wrap is what makes the fill vary. - clippy::cast_possible_truncation -)] - -use ant_node::storage::{ChunkStore, ChunkStoreConfig, LmdbStorage, LmdbStorageConfig}; -use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; -use std::time::Duration; -use tempfile::TempDir; - -/// Chunks the child writes before it is killed. -const CHUNKS: usize = 120; - -/// Deterministic content for chunk `n`. `n` goes in verbatim so no two differ only by a -/// wrap and collapse into one chunk. -fn chunk_bytes(n: usize) -> Vec { - let mut content = vec![0u8; 4096]; - content[..8].copy_from_slice(&(n as u64).to_le_bytes()); - for (i, byte) in content.iter_mut().enumerate().skip(8) { - *byte = ((i.wrapping_mul(17)).wrapping_add(n) % 251) as u8; - } - content -} - -/// Run this test binary again as a child in the mode named by `role`, wait until it has -/// reached the named point, and kill it there. -/// -/// A child process rather than a thread, because the point is to lose everything the -/// process was holding: buffers, in-memory index, locks, half-finished intentions. -/// -/// The wait is a handshake, not a sleep. An earlier version of this slept and hoped, and -/// on a quick machine the child had finished everything before the kill arrived, so the -/// test was checking a clean shutdown while claiming to check a crash. The child now stops -/// at a failpoint inside the write and says so by writing a marker; this waits for the -/// marker and then kills it, so the process always dies at the same point in the same -/// operation. -fn kill_child_at_failpoint(role: &str, root: &Path, failpoint: &str, let_through: u64) -> PathBuf { - let marker = root.join(format!("reached-{role}")); - let _ = std::fs::remove_file(&marker); - - let exe = std::env::current_exe().expect("this test binary"); - let mut child = Command::new(exe) - .arg("--exact") - .arg(role) - .arg("--nocapture") - .arg("--ignored") - .env("ANT_CRASH_TEST_ROOT", root) - .env(failpoint, &marker) - .env( - ant_node::storage::file_store::HALT_AFTER, - let_through.to_string(), - ) - .stdout(Stdio::null()) - .stderr(Stdio::inherit()) - .spawn() - .expect("spawn the child"); - - // Generous, but not unbounded. Without a deadline a failpoint that stopped working - // would hang the job rather than fail it, and a hang says nothing about the code. - let deadline = std::time::Instant::now() + Duration::from_secs(120); - while !marker.exists() { - if let Ok(Some(status)) = child.try_wait() { - panic!("the child exited before reaching the failpoint: {status}"); - } - if std::time::Instant::now() > deadline { - let _ = child.kill(); - panic!("the child never reached the failpoint"); - } - std::thread::sleep(Duration::from_millis(10)); - } - - child.kill().expect("kill the child"); - let _ = child.wait(); - marker -} - -/// Where the child was told to work. -fn child_root() -> PathBuf { - PathBuf::from(std::env::var("ANT_CRASH_TEST_ROOT").expect("the child needs a root")) -} - -/// Child mode: write chunks into a file store until killed. -#[tokio::test] -#[ignore = "child process of a crash test, not run on its own"] -async fn child_writes_until_killed() { - let root = child_root(); - let store = ChunkStore::new(ChunkStoreConfig { - root_dir: root, - disk_reserve: 0, - ..ChunkStoreConfig::default() - }) - .await - .expect("open"); - - // Always a chunk it has not written before, so the kill lands in real work rather - // than in a re-offer of something already on disk. An earlier version cycled the same - // hundred keys and spent almost all its time confirming duplicates. - let mut n = 0usize; - loop { - let content = chunk_bytes(n); - let address = ant_node::client::compute_address(&content); - let _ = store.put(&address, &content).await; - n += 1; - } -} - -/// Child mode: copy a legacy environment into files until killed. -#[tokio::test] -#[ignore = "child process of a crash test, not run on its own"] -async fn child_migrates_until_killed() { - let root = child_root(); - let mut config = ChunkStoreConfig { - root_dir: root.clone(), - disk_reserve: 0, - ..ChunkStoreConfig::default() - }; - config.migration.tick_secs = 1; - config.migration.copier_throttle_mib_per_sec = 0; - config.migration.copier_slack_mb = 0; - config.migration.lock_dir = Some(root); - let store = ChunkStore::new(config).await.expect("open"); - - let shutdown = tokio_util::sync::CancellationToken::new(); - // One chunk at a time, so the kill lands between two of them rather than after the - // whole thing. Deliberately no sleep at the end: a child that finished and then idled - // would let this test pass having crashed nothing. - loop { - let keys = store.legacy_only_keys(); - let Some(key) = keys.first() else { - break; - }; - let _ = store.copy_batch(&[*key], 0, 0, &shutdown).await; - } - panic!("the child copied everything before it was killed, so nothing was interrupted"); -} - -/// Plant a legacy environment holding chunks numbered from `first`, and close it. -async fn seed_legacy_from(root: &Path, first: usize) -> Vec<(usize, [u8; 32])> { - let lmdb = LmdbStorage::new(LmdbStorageConfig { - root_dir: root.to_path_buf(), - verify_on_read: true, - max_map_size: 0, - disk_reserve: 0, - }) - .await - .expect("open legacy"); - let mut keys = Vec::new(); - for n in first..first + CHUNKS { - let content = chunk_bytes(n); - let address = ant_node::client::compute_address(&content); - lmdb.put(&address, &content).await.expect("seed"); - // Paired with its chunk number, because half of these are about to be deleted and - // a bare position in the surviving list no longer says which chunk it is. - keys.push((n, address)); - } - - // Then delete some, which is what makes this look like a real node rather than a - // fresh file. The environment is pinned to its current size for the whole migration, - // so a write during the bridge lands only if there are free pages to land in. On a - // production node there are plenty: this migration exists precisely because deleting - // millions of chunks filled the free list and returned nothing to the filesystem. - // Seeded and never deleted from, the environment would have no room and the bridge's - // second write would never happen, which is not the case worth testing. - let discarded: Vec<(usize, [u8; 32])> = keys.drain(..CHUNKS / 2).collect(); - for (_, address) in &discarded { - lmdb.delete(address).await.expect("make room"); - } - lmdb.wait_idle().await; - keys -} - -/// Child mode: retire the legacy environment, and be killed once it is marked. -/// -/// Everything before the mark is done here rather than in the parent, because the whole -/// point is that the process that wrote the mark is the one that dies. -#[tokio::test] -#[ignore = "child process of a crash test, not run on its own"] -async fn child_retires_until_killed() { - let root = child_root(); - let store = reopen(&root).await; - - let shutdown = tokio_util::sync::CancellationToken::new(); - let keys = store.legacy_only_keys(); - store - .copy_batch(&keys, 0, 0, &shutdown) - .await - .expect("copy every chunk"); - store.wait_idle().await; - store.commit_to_files().expect("commit to the file set"); - store.note_commitment_rebuilt(); - store.note_commitment_rebuilt(); - store.force_migration_state(|state| { - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map_or(0, |d| d.as_secs()); - state.committed_at_unix = Some( - now.saturating_sub(ant_node::storage::migration::MIN_RETIRE_DELAY_HOURS * 3600 + 60), - ); - }); - - let proof = store - .verify_before_retire(0, &shutdown) - .await - .expect("verify before retiring"); - // Parks inside this call, once the environment is renamed aside and marked. - let _ = store - .retire_legacy( - &proof, - &|_: &[u8; 32]| false, - &std::collections::BTreeSet::new(), - ) - .await; - panic!("the child finished retiring without being killed, so nothing was interrupted"); -} - -/// A process killed inside a publish leaves no chunk it cannot serve. -/// -/// The child is stopped at the last moment before the chunk's name exists on disk: on Unix -/// the bytes written to a temporary file with the rename not yet made, off Unix the point -/// before the file is created at all, since that platform writes under the final name -/// because a rename there carries no durability guarantee. The failure this guards against -/// is the same on both: a name outliving its bytes. The index is built from filenames at -/// startup, so a partial file wearing a real chunk name would be advertised, committed to, -/// and unservable. -#[tokio::test] -async fn a_process_killed_mid_publish_leaves_no_chunk_it_cannot_serve() { - let tmp = TempDir::new().expect("temp dir"); - let root = tmp.path().join("node"); - std::fs::create_dir_all(&root).expect("mkdir"); - - // Twenty chunks land before the crash, so the store this reopens has real content in - // it. Stopping the very first write would leave nothing indexed and the loop below - // would pass by iterating over nothing. - let marker = kill_child_at_failpoint( - "child_writes_until_killed", - &root, - ant_node::storage::file_store::HALT_BEFORE_PUBLISH, - 20, - ); - assert!( - marker.exists(), - "the child must have reached the failpoint before it was killed" - ); - - // Reopening is itself part of the assertion: a store that cannot start after a crash - // is a node that cannot start. - let store = reopen(&root).await; - for key in store.all_keys().await.expect("all_keys") { - let served = store.get(&key).await; - assert!( - matches!(served, Ok(Some(_))), - "chunk {} is claimed after a crash but cannot be served: {served:?}", - hex::encode(key) - ); - } -} - -/// The temporary file a killed publish left behind is swept, not indexed. -/// -/// It carries no chunk name, so it can never be served, and leaving it would cost disk -/// for the life of the node. -/// -/// Unix only, because the leftover only exists on Unix. Off Unix the store creates the -/// file under its final name and flushes it, deliberately, since a rename there is not -/// documented to be durable. So there is no temporary file to sweep and the equivalent -/// hazard is different: a real chunk name over bytes that are short or wrong. That one is -/// covered by the store's own tests, which run on every platform, and by the -/// re-hash-everything pass the retirement does before it deletes anything. -#[cfg(unix)] -#[tokio::test] -async fn the_leftovers_of_a_killed_publish_are_swept() { - let tmp = TempDir::new().expect("temp dir"); - let root = tmp.path().join("node"); - std::fs::create_dir_all(&root).expect("mkdir"); - - kill_child_at_failpoint( - "child_writes_until_killed", - &root, - ant_node::storage::file_store::HALT_BEFORE_PUBLISH, - 5, - ); - - let before = temp_files(&root.join("chunks")); - assert!( - before > 0, - "the child should have left a temporary file behind when it was killed" - ); - - let store = reopen(&root).await; - store.wait_idle().await; - assert_eq!( - temp_files(&root.join("chunks")), - 0, - "the store should sweep what an interrupted write left" - ); - drop(store); -} - -/// How many partly-written files are under `chunks_dir`. -#[cfg(unix)] -fn temp_files(chunks_dir: &Path) -> usize { - let Ok(shards) = std::fs::read_dir(chunks_dir) else { - return 0; - }; - shards - .flatten() - .filter_map(|shard| std::fs::read_dir(shard.path()).ok()) - .flat_map(std::iter::IntoIterator::into_iter) - .flatten() - .filter(|entry| { - entry - .file_name() - .to_str() - .is_some_and(|name| !name.chars().all(|c| c.is_ascii_hexdigit())) - }) - .count() -} - -/// Open the store the way a restart would. -async fn reopen(root: &Path) -> ChunkStore { - let mut config = ChunkStoreConfig { - root_dir: root.to_path_buf(), - disk_reserve: 0, - ..ChunkStoreConfig::default() - }; - config.migration.lock_dir = Some(root.to_path_buf()); - ChunkStore::new(config) - .await - .expect("the store must open after a crash") -} - -/// A crash part-way through copying loses nothing: the environment still has everything. -/// -/// The copier is only allowed to drop a key from its list once the file is durably -/// published, so a crash mid-copy costs the work of one chunk, never the chunk. -#[tokio::test] -async fn a_killed_migration_still_has_every_chunk_somewhere() { - let tmp = TempDir::new().expect("temp dir"); - let root = tmp.path().join("node"); - std::fs::create_dir_all(&root).expect("mkdir"); - let keys = seed_legacy_from(&root, 0).await; - - // Ten chunks copied, the eleventh interrupted. An earlier version killed the child - // after a fixed delay, which on a fast runner meant it had copied everything and on a - // slow one meant it had copied nothing; both make this test say something other than - // what it claims. - kill_child_at_failpoint( - "child_migrates_until_killed", - &root, - ant_node::storage::file_store::HALT_BEFORE_PUBLISH, - 10, - ); - - // Interrupted, which is two claims and not one: some chunks copied, and some not. - // Only the upper bound was checked before, so a copier that did nothing at all passed - // as long as everything was still readable from the environment. - let store = reopen(&root).await; - let left = store.legacy_only_keys().len(); - assert!( - left > 0, - "the child was supposed to be killed part-way through, not after finishing" - ); - assert!( - left < keys.len(), - "the child copied nothing, so nothing was interrupted: {left} of {} left", - keys.len() - ); - - for (n, key) in &keys { - let served = store - .get(key) - .await - .expect("read after a crash") - .expect("every seeded chunk must still be readable from one store or the other"); - assert_eq!(served, chunk_bytes(*n), "chunk {n} came back wrong"); - } -} - -/// A crash between the two halves of a dual write does not leave a chunk unprotected. -/// -/// The environment's copy is written first and the file second. A crash in between leaves -/// a chunk only the environment has, and it must be on the copier's list, because a key -/// in neither view is what retirement destroys. -/// -/// The chunks the child writes are deliberately ones the environment does not already -/// hold. An earlier version seeded the same addresses the child then wrote, and the write -/// path skips the environment half for a key that is already legacy-only, so no dual -/// write happened at all and the test proved nothing. -#[tokio::test] -async fn a_crash_between_the_two_halves_leaves_the_chunk_on_the_list() { - let tmp = TempDir::new().expect("temp dir"); - let root = tmp.path().join("node"); - std::fs::create_dir_all(&root).expect("mkdir"); - // Seeded with chunks the child will not write: the child starts at 0 and these are - // far above anything it reaches in the time it has. - seed_legacy_from(&root, 1_000_000).await; - - // The crash lands inside the write of chunk 20, so chunks 0 to 19 completed and 20 - // is the one caught between the two halves. - kill_child_at_failpoint( - "child_writes_until_killed", - &root, - ant_node::storage::file_store::HALT_BEFORE_PUBLISH, - 20, - ); - - let store = reopen(&root).await; - - // Whatever the environment holds and the file store does not is on the list. It is - // derived at open from the two key sets, which is the property that makes a crash - // survivable: re-read from disk, never carried across. - // The specific key, not merely a non-empty list. The environment was seeded with - // unrelated keys, and an earlier version asserted only that something was on the - // list, which those seeds satisfied whether or not a dual write had happened at all. - let interrupted = ant_node::client::compute_address(&chunk_bytes(20)); - let legacy_only = store.legacy_only_keys(); - assert!( - legacy_only.contains(&interrupted), - "the chunk whose file half never landed must be on the copier's list: it reached \ - the environment and nothing else knows about it" - ); - for key in &legacy_only { - let served = store - .get(key) - .await - .expect("read") - .expect("a key on the copier's list must be readable from the environment"); - assert_eq!(ant_node::client::compute_address(&served), *key); - } - - // And nothing the store claims is unservable, from either side of the union. - for key in store.all_keys().await.expect("all_keys") { - assert!( - matches!(store.get(&key).await, Ok(Some(_))), - "chunk {} is claimed after a crash but cannot be served", - hex::encode(key) - ); - } -} - -/// A retirement killed after the mark is finished on the next start, never reopened. -/// -/// The most destructive moment in the whole migration. By the time the mark is written the -/// environment has been renamed aside and the node has already told the network it serves -/// those chunks from the file store. A start that put the directory back would leave the -/// node running two stores again with the disk it came here to free still spent; a start -/// that deleted an *unmarked* directory would destroy a live environment. The mark is what -/// separates the two, and it is written by the process that then dies. -/// -/// Its recovery has unit tests that plant the mark by hand. What those cannot show is that -/// the production path really writes the mark at that moment, before anything is deleted -/// and by the process that then dies. That is what this settles. It does not settle -/// durability: killing a process keeps the kernel page cache, so surviving a kill is not -/// surviving a power cut, which stays a fleet gate. -#[tokio::test] -async fn a_retirement_killed_after_the_mark_is_finished_not_reopened() { - let tmp = TempDir::new().expect("temp dir"); - let root = tmp.path().join("node"); - std::fs::create_dir_all(&root).expect("mkdir"); - let seeded = seed_legacy_from(&root, 0).await; - - kill_child_at_failpoint( - "child_retires_until_killed", - &root, - ant_node::storage::file_store::HALT_AFTER_RETIRE_MARK, - 0, - ); - - // The child died with the directory renamed aside and marked. Nothing had been - // deleted, so this is the state a power cut would leave behind. - let store = reopen(&root).await; - for _ in 0..600 { - if !store.legacy_dir_is_on_disk() && tombstones(&root) == 0 { - break; - } - tokio::time::sleep(Duration::from_millis(100)).await; - } - assert!( - !store.legacy_dir_is_on_disk(), - "the marked environment was put back rather than finished" - ); - assert_eq!( - tombstones(&root), - 0, - "the marked directory is still on disk, so its space was never returned" - ); - - // And every chunk the environment held is still served, from the file store. - for (n, key) in &seeded { - let served = store - .get(key) - .await - .expect("read") - .expect("a chunk must survive an interrupted retirement"); - assert_eq!(served, chunk_bytes(*n), "chunk {n} came back wrong"); - } -} - -/// Directories beside the live environment that a retirement left behind. -fn tombstones(root: &Path) -> usize { - let Ok(entries) = std::fs::read_dir(root) else { - return 0; - }; - entries - .flatten() - .filter(|entry| { - entry.file_type().is_ok_and(|kind| kind.is_dir()) - && entry - .file_name() - .to_str() - .is_some_and(|name| name.starts_with(ant_node::storage::LEGACY_ENV_DIR)) - }) - .count() -} diff --git a/tests/migration_reclaims_disk.rs b/tests/migration_reclaims_disk.rs deleted file mode 100644 index 009e7334..00000000 --- a/tests/migration_reclaims_disk.rs +++ /dev/null @@ -1,390 +0,0 @@ -//! Proof that the migration actually returns disk to the filesystem. -//! -//! This is the claim the whole change exists to make good, and until now it was the one -//! thing the test suite did not check. The unit tests prove the environment is *removed*; -//! that is not the same as the space coming back, which is exactly the mistake that -//! started this work. The fleet deleted 2.29 million chunks, every counter said the -//! chunks were gone, and not one byte returned to the filesystem, because LMDB moves -//! freed pages to its own free list and never shortens the file. -//! -//! So these tests measure the filesystem, not the store's opinion of itself: the size of -//! the data on disk before and after, and the free space the operating system reports. -//! -//! They run on every platform CI covers, which is also the filesystem matrix that matters -//! here: ext4 on Linux, APFS on macOS, NTFS on Windows. - -#![allow( - clippy::unwrap_used, - clippy::expect_used, - clippy::panic, - clippy::missing_panics_doc, - // Test fixtures: every cast here is of a bounded loop counter into a byte, and the - // wrap is what makes the fill vary. - clippy::cast_possible_truncation -)] - -use ant_node::storage::migration::{MigrationPhase, MIN_RETIRE_DELAY_HOURS}; -use ant_node::storage::{ChunkStore, ChunkStoreConfig, LmdbStorage, LmdbStorageConfig}; -use std::path::Path; -use std::sync::Arc; -use tempfile::TempDir; -use tokio_util::sync::CancellationToken; - -/// Chunks to plant. Enough that the environment is meaningfully larger than its own -/// overhead, so "the file shrank" cannot be an accounting artefact. -const CHUNKS: usize = 400; - -/// Bytes per chunk. -/// -/// Sized against the noise, not against the chunk. This test reads what the *filesystem* -/// says is free, which on a shared runner moves for reasons that have nothing to do with -/// it: another job's build, a package cache, an indexer. At 16 KiB a chunk the whole -/// environment came to about 14 MB and the recovery threshold to about 7 MB, which -/// ordinary runner activity can swallow. At 128 KiB it is an order of magnitude clear of -/// that, and 400 chunks still write in a few seconds. -const CHUNK_BYTES: usize = 128 * 1024; - -/// Blocks actually allocated under `path`, in bytes, following no links. -/// -/// Allocated blocks rather than file lengths. A length is what the file claims; blocks -/// are what the filesystem has handed out, and the two part company exactly where this -/// test needs to be careful: a sparse file, a file whose last block is mostly padding, or -/// a file that has been unlinked while something still holds it open. -#[cfg(unix)] -fn allocated_bytes(path: &Path) -> u64 { - use std::os::unix::fs::MetadataExt; - walk(path, &|meta| meta.blocks() * 512) -} - -/// Off Unix, the length is the best the standard library offers. -#[cfg(not(unix))] -fn allocated_bytes(path: &Path) -> u64 { - walk(path, &|meta| meta.len()) -} - -/// Sum `size` over everything under `path`. -fn walk(path: &Path, size: &dyn Fn(&std::fs::Metadata) -> u64) -> u64 { - let Ok(entries) = std::fs::read_dir(path) else { - return std::fs::symlink_metadata(path).map_or(0, |m| size(&m)); - }; - entries - .flatten() - .map(|entry| { - let path = entry.path(); - match std::fs::symlink_metadata(&path) { - Ok(meta) if meta.is_dir() => walk(&path, size), - Ok(meta) => size(&meta), - Err(_) => 0, - } - }) - .sum() -} - -/// What the filesystem says is free, right now. -/// -/// The measurement that cannot be argued with, and the one this test exists for. A path -/// disappearing proves nothing: unlink a file that something still holds open and every -/// name is gone while every block is still spoken for, which is a fair description of the -/// bug that started all this. -fn free_space(path: &Path) -> u64 { - fs2::available_space(path).expect("the filesystem should report its free space") -} - -/// Deterministic content for chunk `n`, filled so it does not compress to nothing. -/// -/// `n` goes in verbatim at the front rather than being folded into the fill, because a -/// fill that wraps makes two different `n` produce the same bytes, and content-addressed -/// storage would then hold one chunk where the test believed it held two. The first -/// version of this test did exactly that and undercounted by a third. -fn chunk_bytes(n: usize) -> Vec { - let mut content = vec![0u8; CHUNK_BYTES]; - content[..8].copy_from_slice(&(n as u64).to_le_bytes()); - for (i, byte) in content.iter_mut().enumerate().skip(8) { - *byte = ((i.wrapping_mul(31)).wrapping_add(n) % 251) as u8; - } - content -} - -/// Plant a legacy environment holding `CHUNKS` chunks and close it. -async fn seed_legacy_environment(root: &Path) -> Vec<[u8; 32]> { - let lmdb = LmdbStorage::new(LmdbStorageConfig { - root_dir: root.to_path_buf(), - verify_on_read: true, - max_map_size: 0, - disk_reserve: 0, - }) - .await - .expect("open the legacy environment"); - - let mut keys = Vec::with_capacity(CHUNKS); - for n in 0..CHUNKS { - let content = chunk_bytes(n); - let address = ant_node::client::compute_address(&content); - lmdb.put(&address, &content).await.expect("seed a chunk"); - keys.push(address); - } - lmdb.wait_idle().await; - keys -} - -/// A store configured to migrate promptly, so a test does not wait out real delays. -fn migrating_config(root: &Path) -> ChunkStoreConfig { - let mut config = ChunkStoreConfig { - root_dir: root.to_path_buf(), - disk_reserve: 0, - ..ChunkStoreConfig::default() - }; - config.migration.tick_secs = 1; - config.migration.copier_throttle_mib_per_sec = 0; - config.migration.copier_slack_mb = 0; - config.migration.lock_dir = Some(root.to_path_buf()); - config -} - -/// Take a settled store all the way through retirement. -/// -/// Drives the steps the driver would, rather than running the driver, so the test does -/// not depend on wall-clock gates it has no business waiting for. The gates themselves -/// are covered by their own tests; what this one is about is the disk. -async fn copy_everything(store: &Arc, keys: &[[u8; 32]]) { - store - .copy_batch(keys, 0, 0, &CancellationToken::new()) - .await - .expect("copy every chunk into the file store"); - assert!( - store.legacy_only_keys().is_empty(), - "every chunk should have been copied" - ); - store.wait_idle().await; -} - -/// Take an already-copied store through retirement, returning the bytes it freed. -/// -/// Separate from the copying so a caller can measure the disk in between, at the peak -/// where both stores hold everything. That is the moment a node is most at risk of -/// filling up, and measuring only the ends would miss it. -async fn retire(store: &Arc) -> u64 { - let shutdown = CancellationToken::new(); - - store - .commit_to_files() - .expect("commit to the file-backed set"); - store.note_commitment_rebuilt(); - store.note_commitment_rebuilt(); - store.force_migration_state(|state| { - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map_or(0, |d| d.as_secs()); - state.committed_at_unix = Some(now.saturating_sub(MIN_RETIRE_DELAY_HOURS * 3600 + 60)); - }); - - let proof = store - .verify_before_retire(0, &shutdown) - .await - .expect("verify before retiring"); - assert!(proof.is_clean(), "verification must pass: {proof:?}"); - - store - .retire_legacy( - &proof, - &|_: &[u8; 32]| false, - &std::collections::BTreeSet::new(), - ) - .await - .expect("retire the legacy environment") -} - -/// The bytes the legacy environment occupied come back to the filesystem. -/// -/// Three measurements, because only the third one settles it: what the filesystem says is -/// free before anything is written, at the peak when both stores hold everything, and -/// after the environment is gone. A test that only watched paths disappear would pass -/// while every block stayed allocated, which is a fair description of the bug that -/// started all this. -/// -/// The numbers are noisy on a shared machine, so the assertion is about the shape: the -/// peak is materially below the start, and the end recovers most of the way back to it. -#[tokio::test] -async fn retiring_the_legacy_environment_returns_its_bytes_to_the_filesystem() { - let tmp = TempDir::new().expect("temp dir"); - let root = tmp.path().join("node"); - std::fs::create_dir_all(&root).expect("mkdir"); - - let payload = (CHUNKS * CHUNK_BYTES) as u64; - // What the reading drifts by here, with nothing of ours happening. Printed rather - // than asserted on: it is what tells whoever reads a failure whether the space did not - // come back or the machine was simply busy. - let quiet = free_space(&root); - tokio::time::sleep(std::time::Duration::from_millis(500)).await; - let drift = quiet.abs_diff(free_space(&root)); - - let free_at_start = free_space(&root); - - let keys = seed_legacy_environment(&root).await; - let environment = root.join("chunks.mdb"); - let environment_blocks = allocated_bytes(&environment); - assert!( - environment_blocks >= payload, - "the seeded environment should have at least the chunk bytes allocated, has \ - {environment_blocks}" - ); - - let store = Arc::new( - ChunkStore::new(migrating_config(&root)) - .await - .expect("open the store"), - ); - assert!(store.has_legacy()); - - // Both stores hold everything: the peak, and the moment a node is most at risk of - // filling its disk. - copy_everything(&store, &keys).await; - let free_at_peak = free_space(&root); - assert!( - free_at_peak < free_at_start, - "holding both copies should have consumed disk" - ); - - let freed = retire(&store).await; - store.wait_idle().await; - assert_eq!(store.migration_phase(), MigrationPhase::FilesOnly); - assert!(freed > 0, "retirement reported no bytes freed"); - - // The deletion runs on a detached thread so the node can serve while it happens. - for _ in 0..400 { - if !environment.exists() && allocated_bytes(&root) < environment_blocks + payload { - break; - } - tokio::time::sleep(std::time::Duration::from_millis(50)).await; - } - assert!( - !environment.exists(), - "the environment directory is still on disk" - ); - - // Dropping the store closes every handle. A file that is unlinked while something - // still holds it open keeps its blocks and shows in no directory, so measuring before - // this point would be measuring the wrong thing. - drop(store); - - // Polled rather than sampled once after a fixed pause. Not every filesystem updates - // its accounting the instant a file goes: btrfs in particular defers it, and a single - // reading taken too early says the space never came back when it is on its way. The - // deadline is what makes this a test rather than a wait. - let free_at_end = wait_for_space(&root, free_at_peak + environment_blocks / 2).await; - - // Only the file store's copy should be left. - let left_on_disk = allocated_bytes(&root); - let file_store_blocks = allocated_bytes(&root.join("chunks")); - let recovered = free_at_end.saturating_sub(free_at_peak); - assert!( - left_on_disk <= file_store_blocks + (payload / 10), - "something other than the file store is still using disk: {left_on_disk} total \ - against {file_store_blocks} in the file store" - ); - - // And the filesystem agrees, measured against the peak rather than against a guess. - // Retiring should hand back most of what the environment was occupying, which makes - // this a statement about the environment's own size rather than about the payload. - assert!( - recovered > environment_blocks / 2, - "retiring recovered {recovered} bytes of an environment occupying \ - {environment_blocks}" - ); - - // And what is left costs roughly one copy rather than two, measured against what the - // file store actually occupies rather than against what the environment did. The two - // are not interchangeable: how much a filesystem spends on four hundred small files - // against one large one is its own business, and btrfs in particular charges very - // differently for the two. Printed as well as asserted, so a number that is drifting - // shows up in the log before it trips anything. - let consumed = free_at_start.saturating_sub(free_at_end); - println!( - "reclaim: environment {environment_blocks} bytes, file store {file_store_blocks}, \ - peak cost {}, end cost {consumed}, recovered {recovered}, ambient drift {drift} \ - in half a second", - free_at_start.saturating_sub(free_at_peak) - ); - assert!( - consumed < file_store_blocks + environment_blocks / 2, - "the filesystem is still down {consumed} bytes with only {file_store_blocks} of \ - file store to account for it, so the environment's space did not come back" - ); - - // Every chunk is still served, read back through a store opened from scratch, which - // is what a restart does. Space recovered by losing data would be no achievement, and - // it is the failure this whole change exists to avoid. - let fresh = store_reopened(&root).await; - for (n, key) in keys.iter().enumerate() { - let served = fresh - .get(key) - .await - .expect("read a migrated chunk") - .expect("a migrated chunk should still be there"); - assert_eq!(served, chunk_bytes(n), "chunk {n} came back wrong"); - } -} - -/// Wait for the filesystem to report at least `wanted` bytes free, and return what it -/// reports at the end. -/// -/// Returns whatever it last saw when the deadline passes, so the caller's assertion is -/// what fails rather than this helper, and the number in the failure is a real reading. -async fn wait_for_space(path: &Path, wanted: u64) -> u64 { - let deadline = std::time::Instant::now() + std::time::Duration::from_secs(60); - loop { - let free = free_space(path); - if free >= wanted || std::time::Instant::now() > deadline { - return free; - } - tokio::time::sleep(std::time::Duration::from_millis(100)).await; - } -} - -/// Open the store again from scratch, which is what a restart does. -async fn store_reopened(root: &Path) -> ChunkStore { - ChunkStore::new(ChunkStoreConfig { - root_dir: root.to_path_buf(), - disk_reserve: 0, - ..ChunkStoreConfig::default() - }) - .await - .expect("the store must reopen after retirement") -} - -/// The file store holds the same payload in less space than the environment did. -/// -/// Not a compression claim: it is that one file per chunk carries no free list and no -/// map overhead, which is the whole reason the space can be returned at all. -#[tokio::test] -async fn the_file_store_holds_the_same_chunks_in_less_space() { - let tmp = TempDir::new().expect("temp dir"); - let root = tmp.path().join("node"); - std::fs::create_dir_all(&root).expect("mkdir"); - - let keys = seed_legacy_environment(&root).await; - let environment_bytes = allocated_bytes(&root.join("chunks.mdb")); - - let store = Arc::new( - ChunkStore::new(migrating_config(&root)) - .await - .expect("open the store"), - ); - store - .copy_batch(&keys, 0, 0, &CancellationToken::new()) - .await - .expect("copy"); - store.wait_idle().await; - - let payload = (CHUNKS * CHUNK_BYTES) as u64; - let file_store_bytes = allocated_bytes(&root.join("chunks")); - assert!( - file_store_bytes >= payload, - "the file store should hold at least the payload: {file_store_bytes} < {payload}" - ); - assert!( - file_store_bytes <= environment_bytes, - "one file per chunk should not cost more than the environment did: \ - {file_store_bytes} > {environment_bytes}" - ); -} diff --git a/tests/migration_shared_volume.rs b/tests/migration_shared_volume.rs deleted file mode 100644 index 8ff79791..00000000 --- a/tests/migration_shared_volume.rs +++ /dev/null @@ -1,512 +0,0 @@ -//! Several nodes migrating on one disk. -//! -//! Operators run many nodes per machine, and during the bridge each one briefly holds two -//! copies of everything it stores. If they all did that at once the disk would fill, which -//! is the failure this migration exists to prevent rather than cause. A lock keyed by the -//! filesystem lets one node at a time do the copying. -//! -//! The lock has a cap on how long a single node may hold it, so one node stuck waiting on -//! its neighbours cannot keep the rest of the machine from ever starting. That cap is -//! hours long by design, so what is checked here is the exclusion itself and the -//! accounting around it, not the cap expiring. - -#![allow( - clippy::unwrap_used, - clippy::expect_used, - clippy::panic, - clippy::missing_panics_doc, - // Test fixtures: every cast here is of a bounded loop counter into a byte, and the - // wrap is what makes the fill vary. - clippy::cast_possible_truncation -)] - -use ant_node::storage::migration::{self, LockAttempt, VolumeLock, MIN_RETIRE_DELAY_HOURS}; -use ant_node::storage::{ChunkStore, ChunkStoreConfig, LmdbStorage, LmdbStorageConfig}; -use std::path::Path; -use std::sync::Arc; -use std::time::Duration; -use tempfile::TempDir; -use tokio_util::sync::CancellationToken; - -/// Chunks each node holds. -const CHUNKS: usize = 60; - -fn chunk_bytes(node: usize, n: usize) -> Vec { - let mut content = vec![0u8; 8192]; - content[..8].copy_from_slice(&(n as u64).to_le_bytes()); - content[8..16].copy_from_slice(&(node as u64).to_le_bytes()); - for (i, byte) in content.iter_mut().enumerate().skip(16) { - *byte = ((i.wrapping_mul(29)).wrapping_add(n).wrapping_add(node) % 251) as u8; - } - content -} - -async fn seed_legacy(root: &Path, node: usize) -> Vec<[u8; 32]> { - let lmdb = LmdbStorage::new(LmdbStorageConfig { - root_dir: root.to_path_buf(), - verify_on_read: true, - max_map_size: 0, - disk_reserve: 0, - }) - .await - .expect("open legacy"); - let mut keys = Vec::new(); - for n in 0..CHUNKS { - let content = chunk_bytes(node, n); - let address = ant_node::client::compute_address(&content); - lmdb.put(&address, &content).await.expect("seed"); - keys.push(address); - } - lmdb.wait_idle().await; - keys -} - -/// One node at a time copies; the others wait rather than piling on. -/// -/// The lock is taken per filesystem, not per node directory. That distinction is the -/// whole point: two nodes are configured with different roots by definition, so a lock -/// beside each root would serialise neither against the other. -#[test] -fn only_one_node_on_a_volume_holds_the_lock() { - let volume = TempDir::new().expect("temp dir"); - let node_a = volume.path().join("node-a"); - let node_b = volume.path().join("node-b"); - let node_c = volume.path().join("node-c"); - for root in [&node_a, &node_b, &node_c] { - std::fs::create_dir_all(root).expect("mkdir"); - } - - // Scoped to this volume directory so the test does not contend with anything else on - // the machine's real filesystem. - let scope = Some(volume.path()); - - let LockAttempt::Acquired(held) = VolumeLock::try_acquire(&node_a, scope) else { - panic!("the first node must take the lock"); - }; - assert!( - matches!(VolumeLock::try_acquire(&node_b, scope), LockAttempt::Busy), - "a second node on the same volume must wait" - ); - assert!( - matches!(VolumeLock::try_acquire(&node_c, scope), LockAttempt::Busy), - "and so must a third" - ); - - drop(held); - assert!( - matches!( - VolumeLock::try_acquire(&node_b, scope), - LockAttempt::Acquired(_) - ), - "the lock must pass on once the first node lets go" - ); -} - -/// A migration context with no network, which is all these tests need. -/// -/// The gates that consult routing have their own tests; what is under test here is the -/// lock, and a node with no view of the network still copies. -fn offline_context() -> migration::MigrationContext { - migration::MigrationContext { - p2p: None, - self_id: None, - self_xor: None, - commitment: None, - replication: None, - sync_state: None, - audit_challenge_coordinator: None, - peer_commitments: None, - close_group_size: 7, - } -} - -/// Two migration drivers on one disk: only one copies at a time. -/// -/// This drives `migration::run`, not `copy_batch`. The copier does not take the volume -/// lock; the driver does, and an earlier version of this test called the copier directly -/// and would have passed with the lock removed from the driver entirely. -#[tokio::test] -async fn two_drivers_on_one_volume_do_not_copy_at_the_same_time() { - let volume = TempDir::new().expect("temp dir"); - let mut stores = Vec::new(); - let mut all_keys = Vec::new(); - - for node in 0..2 { - let root = volume.path().join(format!("node-{node}")); - std::fs::create_dir_all(&root).expect("mkdir"); - let keys = seed_legacy(&root, node).await; - - stores.push(driven_node(volume.path(), &root).await); - all_keys.push(keys); - } - - // Hold the volume before either driver starts, so both are shut out and neither can - // be observed making progress. - let LockAttempt::Acquired(held) = - VolumeLock::try_acquire(&volume.path().join("an-outsider"), Some(volume.path())) - else { - panic!("the outsider must take the lock"); - }; - - let shutdown = CancellationToken::new(); - let drivers: Vec<_> = stores - .iter() - .map(|store| { - tokio::spawn(migration::run( - Arc::clone(store), - offline_context(), - shutdown.clone(), - )) - }) - .collect(); - - // Long enough for several ticks. Neither driver may copy anything while the lock is - // held by somebody else. - tokio::time::sleep(Duration::from_secs(4)).await; - for (node, store) in stores.iter().enumerate() { - assert_eq!( - store.legacy_only_keys().len(), - CHUNKS, - "node {node} copied while another holder had the volume" - ); - } - - // Released: one of them takes it and copies. The other must not, because the holder - // keeps the volume from its first copy through to retiring, rather than handing it - // back between chunks. That is the point of the lock: two nodes copying at once each - // hold two copies of everything, and the disk this migration exists to free is the - // one that fills. - drop(held); - let mut copier = None; - for _ in 0..300 { - if let Some(node) = stores - .iter() - .position(|s| s.legacy_only_keys().len() < CHUNKS) - { - copier = Some(node); - break; - } - tokio::time::sleep(Duration::from_millis(100)).await; - } - let copier = copier.expect("one driver should have taken the volume and started"); - - // Give the other one many ticks to misbehave in. - tokio::time::sleep(Duration::from_secs(3)).await; - let waiting = 1 - copier; - assert_eq!( - stores[waiting].legacy_only_keys().len(), - CHUNKS, - "node {waiting} copied while node {copier} held the volume" - ); - - // And the one that has it finishes copying, checking on every tick that the other has - // still not started. The window being watched is the whole of the first node's copy - // rather than its two ends. - // - // Copying is as far as this one goes. Retirement is gated behind hours of wall clock - // that a test has no business waiting out, so whether the lock spans that half too has - // its own test below. - for _ in 0..600 { - assert_eq!( - stores[waiting].legacy_only_keys().len(), - CHUNKS, - "node {waiting} copied while node {copier} still held the volume" - ); - if stores[copier].legacy_only_keys().is_empty() { - break; - } - tokio::time::sleep(Duration::from_millis(100)).await; - } - shutdown.cancel(); - for driver in drivers { - let _ = driver.await; - } - - assert!( - stores[copier].legacy_only_keys().is_empty(), - "the node holding the volume did not finish copying" - ); - // Both of them, not just the one that went first. Counting only the copier would pass - // for a node that had picked up its neighbour's chunks as well as its own. - for (node, store) in stores.iter().enumerate() { - holds_exactly_its_own(store, node, &all_keys[node]).await; - } -} - -/// A node set up to be driven by `migration::run` on a shared volume. -async fn driven_node(volume: &Path, root: &Path) -> Arc { - let mut config = ChunkStoreConfig { - root_dir: root.to_path_buf(), - disk_reserve: 0, - ..ChunkStoreConfig::default() - }; - config.migration.tick_secs = 1; - config.migration.copier_throttle_mib_per_sec = 0; - config.migration.copier_slack_mb = 0; - // Small enough that copying takes several ticks, so there is a window in which the - // other node could misbehave and be caught, and large enough that the whole thing - // finishes in seconds rather than one chunk per tick. - config.migration.batch_chunks = 8; - config.migration.lock_dir = Some(volume.to_path_buf()); - Arc::new(ChunkStore::new(config).await.expect("open a node")) -} - -/// Every chunk this node seeded is still served, and nothing else is. -async fn holds_exactly_its_own(store: &ChunkStore, node: usize, keys: &[[u8; 32]]) { - assert_eq!( - store.current_chunks().expect("count") as usize, - keys.len(), - "node {node} must hold its own chunks and only its own" - ); - for (n, key) in keys.iter().enumerate() { - let served = store - .get(key) - .await - .expect("read") - .expect("every chunk this node seeded must still be here"); - assert_eq!( - served, - chunk_bytes(node, n), - "node {node} chunk {n} is wrong" - ); - } -} - -/// A node waits for the volume before it retires, not only before it copies. -/// -/// The driver is documented as holding the volume from the first copy through retirement -/// and not handing it back in between. The test above covers the copying half. This one -/// covers the other, which is the half that matters most: retiring means re-reading every -/// chunk in the store to verify it and then deleting an environment, so it is the heaviest -/// the disk gets. A driver that took the lock only for copying would run that pass while -/// eleven neighbours ran theirs. -/// -/// Shaped the same way as the copying test, and for the same reason: an outsider holds the -/// volume first, so the answer does not depend on catching a short window. The node is put -/// in the phase where retirement is the next thing it would do, and then watched for not -/// doing it. -#[tokio::test] -async fn a_node_waits_for_the_volume_before_it_retires() { - let volume = TempDir::new().expect("temp dir"); - let root = volume.path().join("node-0"); - std::fs::create_dir_all(&root).expect("mkdir"); - let keys = seed_legacy(&root, 0).await; - - let store = ready_to_retire(volume.path(), &root, &keys).await; - let shutdown = CancellationToken::new(); - - let LockAttempt::Acquired(held) = - VolumeLock::try_acquire(&volume.path().join("an-outsider"), Some(volume.path())) - else { - panic!("the outsider must take the lock"); - }; - - let driver = tokio::spawn(migration::run( - Arc::clone(&store), - offline_context(), - shutdown.clone(), - )); - - // Several ticks with everything else in place. The environment must still be there. - for _ in 0..40 { - assert!( - store.legacy_dir_is_on_disk(), - "the node retired while an outsider held the volume" - ); - tokio::time::sleep(Duration::from_millis(100)).await; - } - - // Released: now it retires. Without this half the test would pass against a node that - // never retires at all, which is the failure the whole migration exists to avoid. - drop(held); - let mut retired = false; - for _ in 0..600 { - if !store.legacy_dir_is_on_disk() && !store.has_legacy() { - retired = true; - break; - } - tokio::time::sleep(Duration::from_millis(100)).await; - } - shutdown.cancel(); - let _ = driver.await; - assert!( - retired, - "the node never retired once the volume was free, so it was not waiting for it" - ); - - // And it still holds everything it had. Retiring is deleting the old copy, not the - // chunks. - for (n, key) in keys.iter().enumerate() { - let served = store - .get(key) - .await - .expect("read") - .expect("every chunk must survive retirement"); - assert_eq!( - served, - chunk_bytes(0, n), - "chunk {n} is wrong after retiring" - ); - } -} - -/// Open a node whose only remaining migration work is to retire. -/// -/// Copied and committed by hand rather than by waiting for the driver, because the driver -/// gets here by waiting out the shed hold, which is days. Those gates have their own -/// tests; what the caller is about to watch is the volume lock. -async fn ready_to_retire(volume: &Path, root: &Path, keys: &[[u8; 32]]) -> Arc { - let mut config = ChunkStoreConfig { - root_dir: root.to_path_buf(), - disk_reserve: 0, - ..ChunkStoreConfig::default() - }; - config.migration.tick_secs = 1; - config.migration.copier_throttle_mib_per_sec = 0; - config.migration.copier_slack_mb = 0; - config.migration.lock_dir = Some(volume.to_path_buf()); - let store = Arc::new(ChunkStore::new(config).await.expect("open a node")); - - store - .copy_batch(keys, 0, 0, &CancellationToken::new()) - .await - .expect("copy every chunk"); - store.wait_idle().await; - store.commit_to_files().expect("commit to the file set"); - store.note_commitment_rebuilt(); - store.note_commitment_rebuilt(); - store.force_migration_state(|state| { - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map_or(0, |d| d.as_secs()); - // Past the delay that buys the rollback window, so the only thing left between - // this node and deleting its environment is the volume. - state.committed_at_unix = Some(now.saturating_sub(MIN_RETIRE_DELAY_HOURS * 3600 + 60)); - }); - assert!( - store.legacy_dir_is_on_disk(), - "the environment should still be here before the driver runs" - ); - store -} - -/// Two nodes sharing a disk both finish, and neither loses a chunk to the other. -/// -/// Run one after the other, which is what the lock produces. What is checked is that the -/// second node's copy is unaffected by the first having already run on the same -/// filesystem: no shared state, no name collisions, no lock left behind. -/// -/// Copying only. It drives `copy_batch` rather than the driver, so it says nothing about -/// retirement and is not named as if it did: disabling retirement altogether would leave -/// it green. Retirement on a shared volume is -/// [`a_node_waits_for_the_volume_before_it_retires`]. -#[tokio::test] -async fn nodes_sharing_a_volume_do_not_take_each_others_chunks() { - let volume = TempDir::new().expect("temp dir"); - let shutdown = CancellationToken::new(); - - let mut nodes = Vec::new(); - for node in 0..2 { - let root = volume.path().join(format!("node-{node}")); - std::fs::create_dir_all(&root).expect("mkdir"); - let keys = seed_legacy(&root, node).await; - nodes.push((root, keys)); - } - - for (node, (root, keys)) in nodes.iter().enumerate() { - let mut config = ChunkStoreConfig { - root_dir: root.clone(), - disk_reserve: 0, - ..ChunkStoreConfig::default() - }; - config.migration.lock_dir = Some(volume.path().to_path_buf()); - let store = ChunkStore::new(config).await.expect("open"); - - store.copy_batch(keys, 0, 0, &shutdown).await.expect("copy"); - store.wait_idle().await; - - assert!( - store.legacy_only_keys().is_empty(), - "node {node} should have copied everything" - ); - for (n, key) in keys.iter().enumerate() { - let served = store - .get(key) - .await - .expect("read") - .expect("every chunk this node seeded must still be here"); - assert_eq!( - served, - chunk_bytes(node, n), - "node {node} chunk {n} came back wrong" - ); - } - } - - // Neither node picked up the other's chunks, which sharing a filesystem must not - // cause: the stores are separate, only the lock is shared. - let (root_a, keys_a) = &nodes[0]; - let store_a = ChunkStore::new(ChunkStoreConfig { - root_dir: root_a.clone(), - disk_reserve: 0, - ..ChunkStoreConfig::default() - }) - .await - .expect("reopen node 0"); - assert_eq!( - store_a.current_chunks().expect("count") as usize, - keys_a.len(), - "a node must hold its own chunks and only its own" - ); -} - -/// A node that cannot take the lock does not migrate, and does not lose anything either. -/// -/// Waiting is the correct answer: the chunks stay where they are, served from both stores, -/// until the volume is free. -#[tokio::test] -async fn a_node_that_cannot_take_the_lock_keeps_serving() { - let volume = TempDir::new().expect("temp dir"); - let root = volume.path().join("waiting-node"); - std::fs::create_dir_all(&root).expect("mkdir"); - let keys = seed_legacy(&root, 0).await; - - let LockAttempt::Acquired(_held) = - VolumeLock::try_acquire(&volume.path().join("busy-node"), Some(volume.path())) - else { - panic!("the other node must take the lock"); - }; - - let store = driven_node(volume.path(), &root).await; - - // A real driver, running the whole time. Without one this would say only that a store - // opens, and would still pass against a driver that ignored the lock entirely. - let shutdown = CancellationToken::new(); - let driver = tokio::spawn(migration::run( - Arc::clone(&store), - offline_context(), - shutdown.clone(), - )); - - // The store opens and serves regardless of the lock: only the copier waits for it. - assert!(store.has_legacy()); - for _ in 0..30 { - assert_eq!( - store.legacy_only_keys().len(), - CHUNKS, - "the node copied while another held the volume" - ); - for (n, key) in keys.iter().enumerate() { - let served = store - .get(key) - .await - .expect("read") - .expect("a node waiting for the volume still serves everything it holds"); - assert_eq!(served, chunk_bytes(0, n)); - } - tokio::time::sleep(Duration::from_millis(100)).await; - } - - shutdown.cancel(); - let _ = driver.await; -} diff --git a/tests/poc_audit_handler_live.rs b/tests/poc_audit_handler_live.rs index 5f970a08..a8f15e95 100644 --- a/tests/poc_audit_handler_live.rs +++ b/tests/poc_audit_handler_live.rs @@ -360,7 +360,7 @@ async fn committed_key_with_missing_bytes_is_rejected() { /// A successful proof reports what it read and hashed, at a floor per leaf. /// Anchors the rejection case below: it fixes what the measurement means. /// -/// A leaf costs more than its bytes — an LMDB lookup and a blocking-task round +/// A leaf costs more than its bytes — a store lookup and a blocking-task round /// trip are owed whatever its size — and nothing bounds a chunk from below, so /// the charge is `max(content, floor)` per leaf. These test records are 1 KiB, /// well under the floor, which is the case that used to be nearly free: the diff --git a/tests/poc_shutdown_lmdb_drain.rs b/tests/poc_shutdown_lmdb_drain.rs deleted file mode 100644 index 135e8001..00000000 --- a/tests/poc_shutdown_lmdb_drain.rs +++ /dev/null @@ -1,192 +0,0 @@ -//! Regression test for the LMDB drain guarantee of -//! [`ant_node::ReplicationEngine::shutdown`]. -//! -//! ## The vulnerability (pre-fix) -//! -//! Engine tasks race their work against the shutdown `CancellationToken` in -//! `select!`. Dropping the losing future does **not** cancel a -//! `tokio::task::spawn_blocking` LMDB transaction it was awaiting — the -//! closure keeps running on the blocking pool and owns a cloned heed `Env`. -//! `shutdown()` had nothing to wait on for those detached closures (fetch -//! `storage.put`, prune `storage.delete` / `paid_list.remove_batch`, -//! verification `paid_list.insert`), so it could return while the -//! environment was still open. Reopening the same LMDB file with the old -//! `Env` alive in-process is undefined behavior. -//! -//! ## The fix -//! -//! `ChunkStore` (via its file store) and `PaidList` track their blocking tasks in a -//! `TaskTracker`; `shutdown()` awaits `wait_idle()` on both after draining -//! its own tasks. This test parks a chunk-store write inside its blocking -//! closure, drops the awaiter (the exact leak shape), and asserts that -//! `shutdown()` blocks until the write finishes — then proves both LMDB -//! environments reopen cleanly. - -#![allow( - clippy::unwrap_used, - clippy::expect_used, - clippy::panic, - clippy::missing_panics_doc -)] - -use ant_node::payment::{ - EvmVerifierConfig, PaymentVerifier, PaymentVerifierConfig, PriceFloorConfig, -}; -use ant_node::replication::paid_list::PaidList; -use ant_node::storage::{ChunkStore, ChunkStoreConfig}; -use ant_node::{ReplicationConfig, ReplicationEngine}; -use evmlib::{Network as EvmNetwork, RewardsAddress}; -use rand::Rng; -use saorsa_core::identity::NodeIdentity; -use saorsa_core::{NodeConfig as CoreNodeConfig, P2PNode}; -use std::sync::Arc; -use std::time::Duration; -use tokio_util::sync::CancellationToken; - -/// E2E test port range (CLAUDE.md): tests must stay inside 20000-60000, -/// away from production ant-node's 10000-10999. -const TEST_PORT_RANGE_MIN: u16 = 20_000; -/// Upper bound (exclusive) of the E2E test port range. -const TEST_PORT_RANGE_MAX: u16 = 60_000; -/// Attempts to bind a random test port before giving up (mirrors the -/// transient port-bind retry in the e2e testnet harness). -const PORT_BIND_ATTEMPTS: usize = 4; -/// Short probe proving `shutdown()` is still waiting on the parked LMDB op. -const SHUTDOWN_BLOCKED_PROBE: Duration = Duration::from_millis(300); -/// Generous ceiling for `shutdown()` to finish once the op is released. -const SHUTDOWN_COMPLETE_TIMEOUT: Duration = Duration::from_secs(30); -/// Payment cache capacity for the test verifier. -const TEST_PAYMENT_CACHE_CAPACITY: usize = 1000; -/// Rewards address for the test verifier. -const TEST_REWARDS_ADDRESS: [u8; 20] = [0x01; 20]; - -/// Create and start a loopback P2P node on a random port in the test range. -async fn start_p2p_node(identity: &Arc) -> Arc { - let mut last_err = String::new(); - for _ in 0..PORT_BIND_ATTEMPTS { - let port = rand::thread_rng().gen_range(TEST_PORT_RANGE_MIN..TEST_PORT_RANGE_MAX); - let mut config = CoreNodeConfig::builder() - .port(port) - .ipv6(false) - .local(true) - .build() - .expect("build core config"); - config.node_identity = Some(Arc::clone(identity)); - match P2PNode::new(config).await { - Ok(node) => { - node.start().await.expect("start p2p node"); - return Arc::new(node); - } - Err(e) => last_err = e.to_string(), - } - } - panic!("failed to create P2P node after {PORT_BIND_ATTEMPTS} attempts: {last_err}"); -} - -/// A blocking LMDB write whose awaiter was dropped must delay `shutdown()` -/// until it commits, after which both LMDB environments reopen cleanly. -// Holding the gate's write guard across awaits is the point of the test: -// it parks the blocking closure while we probe shutdown(). -#[allow(clippy::await_holding_lock)] -#[tokio::test] -async fn shutdown_waits_for_detached_lmdb_op_and_envs_reopen() { - let temp_dir = tempfile::TempDir::new().expect("create temp dir"); - let root_dir = temp_dir.path().to_path_buf(); - - // The chunk store the engine will hold (and whose env we reopen below). - let storage = Arc::new( - ChunkStore::new(ChunkStoreConfig { - root_dir: root_dir.clone(), - ..ChunkStoreConfig::test_default() - }) - .await - .expect("create storage"), - ); - - let identity = Arc::new(NodeIdentity::generate().expect("generate identity")); - let replication_config = ReplicationConfig::default(); - let payment_verifier = Arc::new(PaymentVerifier::new(PaymentVerifierConfig { - evm: EvmVerifierConfig { - network: EvmNetwork::ArbitrumSepoliaTest, - }, - cache_capacity: TEST_PAYMENT_CACHE_CAPACITY, - close_group_size: replication_config.close_group_size, - local_rewards_address: RewardsAddress::new(TEST_REWARDS_ADDRESS), - price_floor: PriceFloorConfig::default(), - })); - - let p2p = start_p2p_node(&identity).await; - - let (_fresh_tx, fresh_rx) = tokio::sync::mpsc::unbounded_channel(); - let mut engine = ReplicationEngine::new( - replication_config, - Arc::clone(&p2p), - Arc::clone(&storage), - payment_verifier, - identity, - &root_dir, - fresh_rx, - CancellationToken::new(), - ) - .await - .expect("create engine"); - engine.start(p2p.dht_manager().subscribe_events()); - - // Park a put's blocking closure on the test gate, then drop its awaiter - // mid-flight — the exact shape of a select! losing to the shutdown token - // while `storage.put()` awaits `spawn_blocking`. - let content = b"held-open write must block engine shutdown"; - let address = ChunkStore::compute_address(content); - let gate = storage.test_put_gate(); - let parked = gate.write(); - tokio::select! { - biased; - res = storage.put(&address, content) => { - panic!("put must be parked on the test gate, got {res:?}") - } - () = std::future::ready(()) => {} - } - - { - let shutdown_fut = engine.shutdown(); - tokio::pin!(shutdown_fut); - - // shutdown() must not return while the blocking op is still running. - let blocked = tokio::time::timeout(SHUTDOWN_BLOCKED_PROBE, shutdown_fut.as_mut()).await; - assert!( - blocked.is_err(), - "shutdown() returned while a store write was in flight" - ); - - // Release the write; shutdown must now run to completion. - drop(parked); - tokio::time::timeout(SHUTDOWN_COMPLETE_TIMEOUT, shutdown_fut) - .await - .expect("shutdown after releasing the parked op"); - } - - // The detached write committed before shutdown returned. - assert!(storage.exists(&address).expect("exists after shutdown")); - - // Release every reference the test still holds. Per the shutdown - // contract, no engine-spawned work holds the storage or paid list any - // more, so these drops close both environments. - drop(engine); - p2p.shutdown().await.expect("p2p shutdown"); - drop(p2p); - drop(gate); - drop(storage); - - // Both LMDB environments reopen cleanly from the same directory. - let reopened = ChunkStore::new(ChunkStoreConfig { - root_dir: root_dir.clone(), - ..ChunkStoreConfig::test_default() - }) - .await - .expect("reopen chunk store"); - let read_back = reopened.get(&address).await.expect("get after reopen"); - assert_eq!(read_back, Some(content.to_vec())); - - let paid_list = PaidList::new(&root_dir).await.expect("reopen paid list"); - assert_eq!(paid_list.count().expect("paid list count"), 0); -} diff --git a/tests/storage_scale.rs b/tests/storage_scale.rs index 656eedf1..d5ac2627 100644 --- a/tests/storage_scale.rs +++ b/tests/storage_scale.rs @@ -24,7 +24,7 @@ clippy::cast_possible_truncation )] -use ant_node::storage::{FileStore, FileStoreConfig}; +use ant_node::storage::{ChunkStore, ChunkStoreConfig}; use std::path::Path; use std::time::{Duration, Instant}; use tempfile::TempDir; @@ -149,7 +149,7 @@ async fn opening_a_large_store_stays_quick() { let before = resident_bytes(); let opening = Instant::now(); - let store = FileStore::new(FileStoreConfig { + let store = ChunkStore::new(ChunkStoreConfig { root_dir: root.clone(), verify_on_read: true, disk_reserve: 0, @@ -229,7 +229,7 @@ async fn opening_a_large_store_stays_quick() { /// Open a store at `root`, time the scan, and close it again. async fn time_a_scan(root: &Path) -> Duration { let started = Instant::now(); - let store = FileStore::new(FileStoreConfig { + let store = ChunkStore::new(ChunkStoreConfig { root_dir: root.to_path_buf(), verify_on_read: true, disk_reserve: 0, @@ -328,7 +328,7 @@ async fn child_reports_index_memory() { let keys = key_count(); let before = resident_bytes().expect("linux reports this"); - let store = FileStore::new(FileStoreConfig { + let store = ChunkStore::new(ChunkStoreConfig { root_dir: root, verify_on_read: true, disk_reserve: 0, @@ -360,7 +360,7 @@ async fn each_chunk_the_store_writes_costs_one_directory_entry() { let root = tmp.path().join("node"); std::fs::create_dir_all(&root).expect("mkdir"); - let store = FileStore::new(FileStoreConfig { + let store = ChunkStore::new(ChunkStoreConfig { root_dir: root.clone(), verify_on_read: true, disk_reserve: 0, @@ -415,7 +415,7 @@ fn count_entries(path: &Path) -> Entries { // layout marker, and the lock that keeps a second process out. Both are // fixed, so neither grows with the store. Ok(_) - if entry.file_name() == ant_node::storage::file_store::LAYOUT_FILE_NAME + if entry.file_name() == ant_node::storage::chunk_store::LAYOUT_FILE_NAME || entry.file_name() == ".lock" => {} Ok(_) => counted.files += 1, Err(_) => {} @@ -447,7 +447,7 @@ async fn the_startup_scan_does_not_read_chunk_contents() { plant_sized(&root.join("chunks"), keys, chunk); let before = bytes_read().expect("linux reports this"); - let store = FileStore::new(FileStoreConfig { + let store = ChunkStore::new(ChunkStoreConfig { root_dir: root.clone(), verify_on_read: true, disk_reserve: 0,