diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3bf4bc96..95935cae 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,46 +57,13 @@ jobs: version: ${{ env.FOUNDRY_VERSION }} - name: Run unit tests run: cargo test --lib --features test-utils - # Before the e2e suite, deliberately. These are fast and deterministic, and the e2e - # suite flakes on hosted runners for transport reasons that have nothing to do with - # storage. A failing step aborts the job, so anything sequenced after a flaky one - # never reports, which is how these ran on no platform at all for a whole run. - - name: Prove the migration returns disk to the filesystem + - name: Kill a node mid-write and check what survived shell: bash run: | set -euo pipefail - cargo test --test migration_reclaims_disk --features test-utils -- --nocapture --test-threads=1 2>&1 | tee /tmp/reclaims_disk.log - # A target whose required features are not passed is skipped with a - # warning and a zero exit, so a harness can stop running without anyone - # noticing. This is what makes that loud. - grep -qE 'test result: ok\. [1-9]' /tmp/reclaims_disk.log \ - || { echo 'reclaims_disk ran no tests'; exit 1; } - - name: Kill a node mid-migration and check what survived - shell: bash - run: | - set -euo pipefail - cargo test --test migration_crash_safety --features test-utils -- --test-threads=1 2>&1 | tee /tmp/crash_safety.log - # A target whose required features are not passed is skipped with a - # warning and a zero exit, so a harness can stop running without anyone - # noticing. This is what makes that loud. + cargo test --test chunk_store_crash_safety --features test-utils -- --test-threads=1 2>&1 | tee /tmp/crash_safety.log grep -qE 'test result: ok\. [1-9]' /tmp/crash_safety.log \ - || { echo 'crash_safety ran no tests'; exit 1; } - - name: Several nodes migrating on one disk - shell: bash - run: | - set -euo pipefail - cargo test --test migration_shared_volume --features test-utils 2>&1 | tee /tmp/shared_volume.log - # A target whose required features are not passed is skipped with a - # warning and a zero exit, so a harness can stop running without anyone - # noticing. This is what makes that loud. - grep -qE 'test result: ok\. [1-9]' /tmp/shared_volume.log \ - || { echo 'shared_volume ran no tests'; exit 1; } - # Linux only. This one plants a hundred thousand files to measure what a restart - # costs, and the answer it is after is a fleet answer, where every node is Linux. - # The scan itself reads names and nothing else, which is not a platform-specific - # path, and opening a store is covered on all three by the unit tests. Planting that - # many files on the Windows runner would cost minutes of every run to re-measure - # something no node will ever do there. + || { echo 'chunk_store_crash_safety ran no tests'; exit 1; } - name: Startup scan, index memory and inode cost at scale if: runner.os == 'Linux' shell: bash @@ -118,15 +85,7 @@ jobs: run: cargo test --test poc_audit_handler_live --features test-utils - name: Run bootstrap-stall PoC regression marker run: cargo test --test poc_bootstrap_stall --features test-utils - - name: Shutdown waits for writes whose caller has gone - run: cargo test --test poc_shutdown_lmdb_drain --features test-utils - # Runs the storage tests against real ext4, XFS and btrfs rather than whatever the - # runner provides. Deliberately NOT named durability: killing a process and reopening - # the same mounted filesystem keeps the page cache, so this exercises each filesystem's - # syscall, locking, rename and delete behaviour, not its behaviour under power loss. - # That still needs block-device fault injection or a real machine, and remains a fleet - # gate. filesystems: name: Storage on ${{ matrix.fs }} runs-on: ubuntu-latest @@ -148,52 +107,32 @@ jobs: # on whatever the runner happens to give us. ext4 is what most of the fleet is # on; XFS and btrfs are the two the design reasons about separately, btrfs # because it has been observed reordering writes around a rename. - # 3 GiB is ample: these tests use tens of MiB. The scale harness, which is - # the one that needs room, is not in this job. + # 3 GiB is ample: these tests use tens of MiB. truncate -s 3G /tmp/${{ matrix.fs }}.img mkfs.${{ matrix.fs }} -q /tmp/${{ matrix.fs }}.img sudo mkdir -p /mnt/antfs sudo mount -o loop /tmp/${{ matrix.fs }}.img /mnt/antfs sudo chown "$USER" /mnt/antfs df -hT /mnt/antfs - # TMPDIR is what `TempDir::new` uses, so this is what puts the test data on the - # mounted filesystem rather than on the runner's root. - - name: The migration returns disk on ${{ matrix.fs }} + # TMPDIR is what `TempDir::new` uses, so this puts every temporary store these tests + # build on the mounted filesystem rather than on the runner's root. + # + # These used to be the migration harnesses. The migration is gone, and what is left + # worth asking of a filesystem is what the store itself does on it: publish a chunk + # through a temporary and a rename, flush the directory, unlink it again, and rebuild + # an index from the names afterwards. The storage tests do all of that, and running + # them here is what keeps ext4, XFS and btrfs covered now that the harnesses that used + # to cover them have been deleted. They do not measure free space before and after; + # the harness that did was about the migration and went with it. + - name: Storage behaviour on ${{ matrix.fs }} env: TMPDIR: /mnt/antfs shell: bash run: | set -euo pipefail - cargo test --test migration_reclaims_disk --features test-utils -- --nocapture --test-threads=1 2>&1 | tee /tmp/reclaims_disk.log - # A target whose required features are not passed is skipped with a - # warning and a zero exit, so a harness can stop running without anyone - # noticing. This is what makes that loud. - grep -qE 'test result: ok\. [1-9]' /tmp/reclaims_disk.log \ - || { echo 'reclaims_disk ran no tests'; exit 1; } - - name: A node killed mid-write on ${{ matrix.fs }} loses nothing - env: - TMPDIR: /mnt/antfs - shell: bash - run: | - set -euo pipefail - cargo test --test migration_crash_safety --features test-utils -- --test-threads=1 2>&1 | tee /tmp/crash_safety.log - # A target whose required features are not passed is skipped with a - # warning and a zero exit, so a harness can stop running without anyone - # noticing. This is what makes that loud. - grep -qE 'test result: ok\. [1-9]' /tmp/crash_safety.log \ - || { echo 'crash_safety ran no tests'; exit 1; } - - name: Several nodes on one ${{ matrix.fs }} volume - env: - TMPDIR: /mnt/antfs - shell: bash - run: | - set -euo pipefail - cargo test --test migration_shared_volume --features test-utils 2>&1 | tee /tmp/shared_volume.log - # A target whose required features are not passed is skipped with a - # warning and a zero exit, so a harness can stop running without anyone - # noticing. This is what makes that loud. - grep -qE 'test result: ok\. [1-9]' /tmp/shared_volume.log \ - || { echo 'shared_volume ran no tests'; exit 1; } + cargo test --lib --features test-utils storage:: 2>&1 | tee /tmp/storage.log + grep -qE 'test result: ok\. [1-9]' /tmp/storage.log \ + || { echo 'the storage tests ran nothing'; exit 1; } doc: name: Documentation @@ -221,7 +160,8 @@ jobs: - name: Build release (no logging) run: cargo build --release --no-default-features # The crash harness drives the store through a failpoint that parks the process - # forever on an environment variable. It is compiled only under `test-utils`, which + # forever on an environment variable, and the store's own tests use the same one. It + # is compiled only under `test-utils`, which # is not a default feature and is not passed by the release workflow, so a shipped # binary does not contain it. This proves that rather than trusting it: the variable # name is a string literal, so it survives into the binary whenever the code that diff --git a/Cargo.lock b/Cargo.lock index 27a074bf..f43c9633 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -848,13 +848,13 @@ dependencies = [ "mimalloc", "objc2", "objc2-foundation", - "page_size", "parking_lot", "postcard", "proptest", "rand 0.8.8", "reqwest", "rmp-serde", + "rustix", "saorsa-core", "saorsa-pqc", "saorsa-transport", diff --git a/Cargo.toml b/Cargo.toml index 03efec06..e4ef2f81 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -49,7 +49,8 @@ xor_name = "5" lru = "0.16.3" parking_lot = "0.12" # Efficient mutex for cache -# Storage - LMDB via heed for content-addressed chunk store +# LMDB via heed for the paid-key list. The chunk store is one file per chunk and does +# not use it. heed = "0.22" blake3 = "1" @@ -102,8 +103,6 @@ sha2 = "0.10" # Cross-platform file locking for upgrade caches fs2 = "0.4" -# System page size (for LMDB map alignment during resize) -page_size = "0.6" # Protocol serialization postcard = { version = "1.1.3", features = ["use-std"] } @@ -116,6 +115,11 @@ self_encryption = { version = "0.36", optional = true } [target.'cfg(unix)'.dependencies] libc = "0.2" +# Safe wrappers for `openat`/`unlinkat`, so the storage-migration cleanup can delete relative +# to a directory handle it opened `O_NOFOLLOW` rather than by re-resolving a path it has +# already checked. Adds nothing to the build: `tempfile`, a direct dependency, already pulls +# this exact crate and version in with the `fs` feature on. +rustix = { version = "1", features = ["fs"] } [target.'cfg(windows)'.dependencies] self-replace = "1" @@ -132,20 +136,6 @@ serial_test = "3" # Exercise connected peers that never announce a DataChannel to the listener. webrtc-stack = { package = "webrtc", version = "0.14" } -# Proves the migration returns disk to the filesystem, which is the claim the whole -# change exists to make good. Needs the test-only migration-state accessor. -[[test]] -name = "migration_reclaims_disk" -path = "tests/migration_reclaims_disk.rs" -required-features = ["test-utils"] - -# Kills a real child process part-way through writing and migrating, then checks what -# survived. The automatable half of the power-loss gate. -[[test]] -name = "migration_crash_safety" -path = "tests/migration_crash_safety.rs" -required-features = ["test-utils"] - # Startup scan time, index memory and inode cost at scale. Regression gates, not # benchmarks; ANT_SCALE_KEYS raises the count for a deliberate larger run. [[test]] @@ -153,11 +143,11 @@ name = "storage_scale" path = "tests/storage_scale.rs" required-features = ["test-utils"] -# Several nodes migrating on one disk: the volume lock, and that each finishes with its -# own chunks and only its own. +# A process killed part-way through a write: the store is whole or absent, never +# half-indexed, and what the interrupted write left behind is swept. [[test]] -name = "migration_shared_volume" -path = "tests/migration_shared_volume.rs" +name = "chunk_store_crash_safety" +path = "tests/chunk_store_crash_safety.rs" required-features = ["test-utils"] # E2E test infrastructure (run with --features test-utils) @@ -176,7 +166,7 @@ path = "tests/poc_commitment_audit_attacks.rs" required-features = ["test-utils"] # Live responder-handler tests for the v12 audit. Use -# LmdbStorageConfig::test_default(), gated on test-utils. +# ChunkStoreConfig::test_default(), gated on test-utils. [[test]] name = "poc_audit_handler_live" path = "tests/poc_audit_handler_live.rs" @@ -197,14 +187,6 @@ name = "poc_price_floor_live" path = "tests/poc_price_floor_live.rs" required-features = ["test-utils"] -# Shutdown/LMDB-drain regression: `ReplicationEngine::shutdown()` must not -# return while a detached LMDB blocking op is still running. Uses the -# test-only storage put gate, so it requires the test-utils feature. -[[test]] -name = "poc_shutdown_lmdb_drain" -path = "tests/poc_shutdown_lmdb_drain.rs" -required-features = ["test-utils"] - [[test]] name = "webrtc_direct_devnet" path = "tests/webrtc_direct_devnet.rs" diff --git a/config/production.toml b/config/production.toml index 72e82112..f1f1ac61 100644 --- a/config/production.toml +++ b/config/production.toml @@ -46,62 +46,14 @@ enabled = true # Verify content hash on read verify_on_read = true -# Maximum size in GiB of the legacy LMDB store, while one still exists -# (0 = derive it from available disk). Retired along with LMDB itself. -db_size_gb = 0 - -# --- Moving off the legacy LMDB chunk store --- -# -# Chunks are now one file each, under {root_dir}/chunks/. A node that still has a -# chunks.mdb copies it into files in the background, then deletes it whole, which is the -# only moment LMDB's disk comes back. -# -# The two release-level switches (whether to delete the old store, and whether audits -# still penalise) belong to the build, not to this file, so they are deliberately absent. -[storage.migration] -# Run the copier. Turning this off leaves both stores in place forever and never -# returns the old store's disk. -enabled = true - -# Also write new chunks to the legacy store while it exists, so a fleet rollback to an -# older build cannot lose a chunk uploaded during the migration. -dual_write_legacy = true - -# Allow a node that cannot fit its chunks to give up the ones it is furthest from. -# -# Whatever this is set to, a chunk is only ever given up when the node is near the back of -# its group for it, its close group has received the node's reduced commitment, AND all but -# one of that group has cryptographically proven it holds a copy. A node that cannot show -# all three keeps both stores and asks for more disk. Turn this off if you would rather add -# disk than have the node give anything up at all. -allow_shed = true - -# Hours after this build first starts before a node may give anything up, so peers on -# older builds have upgraded and stopped penalising it for doing so. -shed_hold_hours = 72 - -# Hours between one migration wave opening and the next. -# -# A close group is split into waves so only two of its members give chunks up at a time. -# If all seven went together none could prove to the others that a copy survived, and the -# group would deadlock waiting on each other. A node with room to copy everything does not -# wait for a wave: it is never unable to serve, so it is not part of that problem. -wave_hours = 24 - -# Hours between a node committing to what it will keep and deleting the old store. -# Never shorter than 4: that is what the answerability window needs. -retire_delay_hours = 4 - -# Free space, in MiB, the copier leaves untouched on top of disk_reserve_mb. -copier_slack_mb = 2048 - -# Copy rate ceiling, in MiB/s. Keep it modest: an unthrottled copier competing with the -# audit responder for disk turns a storage migration into an audit incident. -copier_throttle_mib_per_sec = 32 - # --- Upgrade --- [upgrade] -enabled = false +# There is no `enabled` setting, and there never was one. This file used to carry +# `enabled = false`, which serde ignored: `UpgradeConfig` has no such field, so every node +# reading this was self-upgrading while its own configuration appeared to say otherwise. +# Removed rather than annotated, because a line that does nothing is worse than an absent one +# — and this one said the opposite of the truth about the mechanism a release rollback would +# have to go through. channel = "stable" check_interval_hours = 1 github_repo = "WithAutonomi/ant-node" diff --git a/deploy/scripts/spawn-nodes.sh b/deploy/scripts/spawn-nodes.sh index 445bca62..49f741c4 100755 --- a/deploy/scripts/spawn-nodes.sh +++ b/deploy/scripts/spawn-nodes.sh @@ -68,15 +68,6 @@ fi # Create directories mkdir -p "$BASE_DIR" "$LOG_DIR" -# The per-volume migration lock. Every node on this host shares it and nothing else, so -# they can serialise their copies off LMDB without being able to reach each other's data. -# It needs its own directory because PrivateTmp=true below gives each unit a /tmp of its -# own, and the node's default lock location is in there: without this every node takes a -# lock nobody else can see, all of them start copying at once, and the host runs out of -# space with several half-finished migrations on it. -LOCK_DIR="${BASE_DIR%/*}/migration" -mkdir -p "$LOCK_DIR" - # Create ant user if not exists if ! id -u ant &>/dev/null; then useradd -r -s /bin/false ant || true @@ -99,8 +90,6 @@ for i in $(seq 0 $((NODE_COUNT - 1))); do # Create node directory mkdir -p "$NODE_DIR" chown ant:ant "$NODE_DIR" - chown ant:ant "$LOCK_DIR" - chmod 0750 "$LOCK_DIR" # Create systemd service cat > "/etc/systemd/system/$SERVICE_NAME.service" <> /etc/security/limits.conf diff --git a/docs/adr/ADR-0022-remove-the-lmdb-chunk-store.md b/docs/adr/ADR-0022-remove-the-lmdb-chunk-store.md new file mode 100644 index 00000000..2b7159ae --- /dev/null +++ b/docs/adr/ADR-0022-remove-the-lmdb-chunk-store.md @@ -0,0 +1,648 @@ +# ADR-0022: Remove the LMDB Chunk Store + +- **Status:** Proposed +- **Date:** 2026-08-28 +- **Decision owners:** Anselme Gaeremynck +- **Reviewers:** David Irvine, Chris O'Neil, Mick van der Most van Spijk +- **Supersedes:** none +- **Superseded by:** none +- **Related:** ADR-0014 (one file per chunk, and retiring LMDB), which this completes + +## Context + +Moving chunks off LMDB shipped as three releases, because the penalty for not holding a +close-group chunk is the *auditor's* decision: a node that has to give chunks up cannot stop +its peers punishing it for that. So the peers stopped first. + +1. **First:** suspend that one penalty. +2. **Second:** copy every chunk into a file of its own, then delete `chunks.mdb`. ADR-0014. +3. **Third:** this one. + +ADR-0014 describes the third release as flipping the switch back and nothing more. What +actually has to happen is larger, and two parts of it are decisions rather than clean-up. + +## Decision + +**Do not restore the penalty here.** It stays suspended for one more release, and the +release after this one turns it back on. The reasoning is in *The penalty is restored by the +release after this one* below: a node that was away while the migration ran arrives here +holding a store this build cannot read, and restoring the accusation in the release that +stranded it would slash it for a state it had no chance to leave. + +The switch itself stays. The process-wide atomic, the `ANT_SUSPEND_UNHELD_CHUNK_PENALTY` +override and the startup announcement are not migration machinery: they are one release-level +policy that several audit paths have to obey identically, and the release that restores a +penalty is exactly the one most likely to need it undone in a hurry. Removing them would discard the +cheapest lever at the moment it is most useful. A test now pins the shipped value, because +the existing tests set the switch both ways on purpose and so could never notice which way +it was compiled. + +**Delete the LMDB chunk store and the migration, and keep the name `ChunkStore`.** There is +one store. It is one file per chunk, it lives in `src/storage/chunk_store.rs`, and it is +called `ChunkStore` because that is what it is and what every caller already called it. The +type that used to present two stores as one is gone with the second store. + +`heed` stays in the dependency list. The paid-key list has its own LMDB environment, which +this decision does not touch. + +**A node that still has an unretired `chunks.mdb` starts anyway, and clears up what it can +prove is finished with.** This is the part worth arguing, and two earlier drafts of this +decision got it wrong in opposite directions. + +That draft refused to start, and the reasoning was not silly. The chunks in that environment +are unreachable to this build, but the commitment the node published before the upgrade +*claimed* them, and a commitment stays answerable to its neighbours for three hours +(`GOSSIP_ANSWERABILITY_TTL`, which is `(RETAINED_GOSSIPED_COMMITMENTS + 1)` rotations). The +accusation the first release suspended was "you did not have a chunk you were supposed to +hold"; the commitment-bound subtree audit was never suspended in any release, precisely +because it rests on a signed claim. So a node that starts half-migrated does spend hours +failing audits, at the full weight, on the one lane that always counted. + +Three things make refusing the worse answer anyway. + +**A node that refuses serves nothing.** Not the chunks it cannot read, and not the far larger +number it migrated perfectly well. The bounded harm being avoided is a few hours of trust +penalty; the harm being accepted is the entire node, and not for a few hours. + +**There is no recovery.** The obvious instruction, "put the previous build back and let it +finish", cannot be carried out. `Node::build_upgrade_monitor` is called unconditionally and +`UpgradeConfig` has no field that switches it off, so a node put back on the previous release +polls, finds this one, and takes it. There is nothing to set, in a config file or on the +command line, that stops it. On the deployed unit (`Restart=always`, `RestartSec=10`) a +refusing node is a ten-second restart loop with no health check and no automatic rollback, and +it stays in it until a person intervenes. An instruction nobody can follow is not a +mitigation. + +(How quickly it is taken depends on the deployment. The unit in `deploy/terraform` runs the +node as `ant` under `ProtectSystem=strict` with write access only to its own directory, and +the binary lives in `/usr/local/bin`, so the in-process replacement cannot complete there at +all. That is a separate problem with that unit, not a way to hold a node on an old release: +nothing in the node consents to staying.) + +**The population that reaches here is not the one refusing would protect.** A node that has +been offline long enough to miss the previous release entirely has no retained root at any +peer — the answerability window is three hours and its close group has long since re-placed +what it held — so it takes no *commitment-bound* penalty, which is the lane refusing was +protecting it from. The node that is exposed on that lane is one that was running and +unmigrated when this release landed, and that is exactly the population the previous release's +fleet signal exists to count and wait out. + +It is not penalty-free, and it is worth being exact about what remains rather than rounding it +to nothing. This release does NOT restore the close-group unheld-chunk penalty; that waits for +the release after it, for the reason given below. What it cannot hold off is the +commitment-bound audit, which is a separate lane and is enforced in every release. A node +carrying an old store it cannot read is a node +with that much less disk, and a node short of disk fails to take on the chunks it is +responsible for and is penalised on that lane like any other full node. That is not a penalty +for having migrated badly; it is the ordinary consequence of a full disk, arriving through a +directory that is full of nothing useful. The node is told exactly that, once, by name, and +the remedy is the operator's: add disk, or delete the directory once its contents are known to +be elsewhere. + +Refusing to start would not have spared it either. It would have had the same disk and none of +the service. + +So this build starts, and it clears up **only what is provably finished with**. + +Deleting whatever it finds was the second draft, and it was also wrong. The upgrade monitor +picks the newest eligible release rather than the next one, so a node that was offline +through the previous release arrives here with every chunk it owns in that environment and +nothing at all in the file store. Deleting that is destroying data that may have no other +copy, in order to reclaim disk. + +The evidence that separates the two is the mark the previous release wrote *inside* the +directory before it deleted anything. A directory carrying `RETIRED` is one whose retirement +gates were all satisfied, and it is pure cost. So is an empty one, which is what a cleanup +interrupted between emptying a tombstone and removing it leaves. Those go, and their disk comes +back. + +**Be exact about what those gates were, because "its contents were copied out" is not it** and +an earlier draft of this record said so four times. Retirement cleared a directory on two +different grounds, and only the first is a local copy: + +- every chunk the node was **keeping** was copied into the file store and re-hashed there, byte + for byte, before the mark went down; and +- every chunk the node was **shedding** was proven to be held by its close group — all but one + of them answering a possession challenge with a cryptographic proof, after the reduced + commitment had been delivered — and was then deliberately *not* copied. The pre-retirement + verification pass skips exactly these keys, because re-hashing a chunk the node is giving up + into a file store that is not going to keep it would defeat the point of shedding it. + +So a marked directory can, entirely legitimately, contain bytes that are in no file store on +this node and never will be. The previous release was about to delete those bytes itself; this +one finishes that. The safety argument for them is the close group's proofs, not a local copy, +and it is the network that holds them afterwards. + +| what is on disk | what the cleanup does | what the node then serves | disk back | +|---|---|---|---| +| nothing, or a root that does not exist yet | nothing | whatever it stores from here | n/a | +| a leftover carrying its `RETIRED` mark | remove it in the background, report the space once it is back | its file store — everything it kept, and nothing it shed | yes | +| a leftover with nothing in it | remove it | its file store | yes | +| a leftover with chunks in it and no mark | **keep it**, name it once | its file store only — **not** what is in that directory | no | +| a leftover whose contents or mark cannot be established | keep it, name it once | its file store only | no | +| a link at either name | keep it, name it once | its file store only | no | +| a name retirement never created | ignore it entirely | its file store | no | + +The rows that keep something are the point, and the third column is the part it would be easy +to round off. **Kept is not the same as available.** There is no LMDB reader in this build, so +a node that keeps a directory keeps its bytes on disk and cannot serve one of them. For a node +that was part-way through the migration that is the tail it had not copied yet; for a node that +skipped the previous release altogether it is everything it holds, and such a node serves only +what it refetches from here on, exactly as a new node would, while its old bytes sit there +costing disk. + +That is worth being plain about because it is the whole price of the decision: the data is +kept so that it is still there to be recovered by hand or by a future build, not because this +release can do anything with it. What the node gets is its own service back; what it does not +get is that disk, which is the honest cost of not being able to prove the directory is safe to +delete. + +Two things are never done, both because a name is not evidence of what is behind it. + +A **link is neither followed nor unlinked**. What is behind it is on storage this node does +not own: following it would delete somebody else's data, and unlinking it would throw away +the only record of where that data went. + +Only the **exact names** the previous release created are considered: `chunks.mdb`, +`chunks.mdb.retired`, and `chunks.mdb.retired.` for `n` in `1..=64` written without a +leading zero. An earlier draft matched by prefix and by "all digits", which would have claimed +a `chunks.mdb.retired-keep-this` an operator put there, and `.007` and `.999999`, which +retirement cannot have produced. The suffix is parsed and written back out so it has to match +itself, and there is a test that plants every near miss. + +The deletion runs on **one** background thread and nothing waits for it. `remove_dir_all` over +a store with millions of files runs for minutes and must not be on the startup path. One +thread rather than one per directory, and they are deleted in turn: the names this release +recognises are the live directory, the unnumbered tombstone and sixty-four numbered ones, so a +root that has been through enough restore cycles can present sixty-six at once, and a thread +each would put sixty-six concurrent recursive deletions on the disk that is also serving +chunks, at the moment a node is starting. Nothing is waiting on them, so doing them in turn +costs nothing that matters. It deletes in place: there is nothing to get out of the way, because the directory holds nothing this node is answerable for and this +build has no code that would read it if it did. An earlier draft renamed first and could run +out of names to rename to, at which point it stopped removing anything at all, permanently. +The space is only reported as returned once the deletion has actually finished, because a +number that is wrong for the next several minutes is worse than no number. + +**The mark has to be the file the previous release wrote**, not merely something at that +name. It is written with `create_new`, so it is always an ordinary file; a directory, a link +or a FIFO wearing the name proves nothing, and this is the answer that authorises deleting +every chunk underneath it. Testing existence alone would let anything at that path clear an +unmigrated store for deletion. + +**The mark is believed, and not re-checked against anything.** It is worth stating as an +assumption rather than leaving it to be inferred, because it is the one that authorises every +deletion here. `RETIRED` records that the *previous release* satisfied both gates above — the +copy-and-re-hash for what it kept, the close-group possession proofs for what it shed. This +build cannot confirm either independently: it cannot re-run a possession challenge for keys it +cannot enumerate, and it has no LMDB reader, so it cannot compare what is in the directory against +what is in the file store, and no cheaper check is available — a file store that opens is not +evidence that it holds any particular key, and counting keys proves nothing about which ones. + +So the mark is taken as final. For every state the previous release can actually produce, that +is correct: it wrote the mark after the copy and the re-hash, and a marked directory in this +release is one whose deletion was interrupted. The state it is wrong for is one no release +produces — an operator restoring an old marked directory alongside a file store that is not the +one it was retired against, or putting a file called `RETIRED` inside an environment by hand. +This release will delete such a directory. That is accepted: the alternative is to keep every +marked leftover for ever, which returns no disk on any node and defeats the release, and the +states that would be protected are ones a person constructed. **An operator restoring a backup +of `chunks.mdb` must not leave the `RETIRED` file in it.** + +**The mark is removed last.** `remove_dir_all` gives no promise about the order it unlinks +things in, and if the mark went before the chunks did and the process stopped there, the next +start would find an unmarked directory with data in it, decide it might be an unmigrated +store, and keep it for good. It is not one, but nothing on disk would say so any more, and +the node would report itself unfinished to the whole network for as long as it lived. So the +contents go, then the mark, then the directory: at every point either the mark is still there +and the next start resumes, or the directory is empty, which is also finished with. + +**None of it happens until the file store has actually opened.** "Finished with" means +retirement's gates were met, and for everything the node kept that means the chunks are in the +file store, which is only true if the file store is there to hold them. +An earlier draft ran this as soon as the root was known, which put the deletion in front of a +constructor that can still fail on an unreadable layout, a directory it cannot create, or a +lock another process has not let go of — and a node that lost both stores that way had +nothing to go back to. Waiting costs nothing in what this node reports: its user agent is +fixed when the transport is built, a moment earlier, but everything removed here is a leftover +the signal already reads as finished with — carrying the mark or being empty is both what +makes it removable and what makes it harmless — so the node announces `files` whether the +deletion has finished, is still running, or has not started. + +Nothing here can fail. `clean_up` returns `()`. Every way a filesystem can disappoint it ends +in a node that runs, a warning that names the directory, and disk that has not come back — +which is a worse place than success and a far better one than a restart loop. + +Nothing here can leave a node with no store: the deletion runs only after the store that +replaced it has opened, and only over directories retirement had already cleared — everything +kept copied and re-hashed, everything shed proven held by the close group. A node that +never opens one, because storage is switched off, deletes nothing at all — it has established +nothing about where those chunks went, and it has no use for the disk either. + +Said precisely, because the looser version of it is not true: **nothing here vetoes a start**. +That is not the same as "every node starts". The file store is built before this runs, and a +store that cannot open — an unreadable layout, a directory it cannot create, a lock another +process holds — still stops the node, exactly as it did in the release before this one. What +this release removes is the *other* reason a node could fail to start, the one its first draft +introduced: being refused for what it was found carrying. + +**What may be deleted is decided by the previous release's own classifier**, not by a second +reading of the same directory. The signal ADR-0014 put on the wire reports a node as finished +when nothing under its root is holding chunks or unreadable, and this release is published on +the strength of that count; so the cleanup calls that same function rather than carrying its +own idea of which directories are finished with. Two classifiers could drift, and either +direction is a fault: one would let a node delete a directory it was still reporting as +unfinished, the other would leave it reporting `files` while paying for the disk for ever. +There is a test that stages every shape a root can present and checks both directions of that +correspondence, so a re-introduced private classifier fails there rather than on a fleet. + +**The deletion is made against a directory handle, not against a path.** An earlier draft of +this decision named a path for the checks and named it again for the unlinking, and accepted the +window between them on the ground that whoever could win it already had write access to the data +directory. That reasoning was wrong, and independent review caught it: it covers deleting things +*inside* the data directory and says nothing about the variant that reaches outside. `read_dir` +follows a link. Point that name at a target elsewhere on the disk in the window, and the node +empties the target instead — and the node reaches far more of the filesystem than the actor who +moved the name does, and on many installations runs as root. That is privilege escalation, not +one more way to lose a chunk. + +So on Unix the directory is opened once, `O_NOFOLLOW | O_DIRECTORY`, and every unlink is made +against that handle with `unlinkat` — the mark's included, so the handle is not dropped and the +path re-opened for the last step, which was a second window and is gone. A link cannot produce a +handle, so no unlink can be redirected through one. The pattern and the reasoning are already in +this codebase: `open_regular` refuses a link and a FIFO on the handle rather than on the path, +for the same class of reason. + +**A subdirectory is refused, never entered**, and that is a second decision rather than a detail +of the first. `O_NOFOLLOW` declines a trailing symlink and says nothing about a mount point, +which `openat` walks into without complaint; a bind mount can also make a cycle. A retired chunk +environment is flat — two files and a mark — so a directory inside one is already something this +build does not understand, and descending into it risks emptying a filesystem that has nothing +to do with this node. Refusing keeps that shut, and it also leaves no recursion to bound and no +descriptor chain to exhaust. For the same reason a directory holding more entries than any +leftover ever has is refused rather than read into memory: a corrupt or hostile one should not +be able to take the node down through the cleanup that was meant to return it some disk. + +**And the mark is not removed on the strength of a stale listing.** The names are read once and +unlinked one by one, and a listing is not a snapshot: anything created while that loop runs is +not in it and is still there at the end. So the handle is asked once more, immediately before +the one irreversible step, and the mark stays if the answer is no. Without that the ordering +protects nothing in exactly the case it exists for — the mark goes, something is left, and every +later start reads an unmigrated store. + +**The type test on the mark masks with `S_IFMT`.** `S_IFLNK` and `S_IFSOCK` each contain every +bit of `S_IFREG`, so testing `st_mode & S_IFREG` accepts a symlink or a socket wearing the +mark's name — and that answer is what authorises deleting a store nothing migrated. This was +written wrongly first and caught by review; there is a test for both types now. + +The directory itself still goes by path, and that is safe on its own terms: `rmdir` refuses a +symlink, so a swapped name fails the call rather than following it. + +What is pinned by test is the primitive, not the interleaving, and the difference is worth being +exact about. The race cannot be staged without instrumenting the deleter. What the tests assert +is that a link never yields a handle, and that a *marked* target behind a link is untouched — +marked deliberately, because an unmarked one is refused by the gates even by a deleter that +follows links, so testing against one would pass while proving nothing. Both fail if `O_NOFOLLOW` +is dropped. + +**Two things this still does not close, both stated rather than half-fixed.** Off Unix there is +no `unlinkat`, so that build keeps the path-based deleter and its window; its exposure is what +the previous release's deleter already had. And on either build the empty-directory case cuts +both ways within the data directory: an entry created inside a directory this found empty is +deleted with it, and an entry created after the mark has gone leaves an unmarked directory with +something in it, which every later start then keeps for good. + +### What this release does NOT delete + +The cleanup matches exactly `chunks.mdb`, `chunks.mdb.retired` and `chunks.mdb.retired.` +for `n` in `1..=64`, and nothing else. Two things under the node root are deliberately +outside that set and must stay outside it: + +- **The migration marker.** Small, harmless, and useful evidence if a node's history is ever + in question. +- **Anything else an operator put there.** A prefix match would claim a + `chunks.mdb.retired-keep-this`, which is why names are matched exactly rather than by + prefix. + +That the cleanup does not touch these today is a property of the exact-name match, not an +accident, and it is stated here because a future widening of that match would be a silent +data loss rather than an obvious one. + +### Values over the size ceiling are not chunks, and none is preserved + +This was argued both ways across the three releases, and one of the earlier answers — preserve +them to a sidecar rather than destroy them — was wrong. It is settled here so that a later +reader does not reopen it from the sidecar's remains. + +**A chunk is at most 4 MB.** `MAX_CHUNK_SIZE` has been `4 * 1024 * 1024` since ant-protocol's +first commit, and every released path by which data enters a node from the network enforces it +before anything is stored: the protocol handler on a paid store, and replication on both the +receive and the fetch path. **So no value over the ceiling has ever entered this network as a +chunk through released node code.** Something planted on a disk by hand is a separate matter and +is refused on read rather than served. It does not follow that none can exist on a disk — one can, and the next paragraph says +how — only that anything which does is not a chunk: not data with a copy elsewhere, not data +whose owner is waiting for it, and nothing any peer would accept or serve. + +**The one way such a value could reach a disk was ours.** Not the network's. The bridge's +public `ChunkStore::put` wrote to the legacy environment *first* — `LmdbStorage::put` has no +size ceiling, deliberately, because the store it wrote to was being abandoned and its verdict +was not allowed to refuse chunks the file store had room for — and only then offered the same +bytes to the file store, which refused them for size. The bridge then recorded the key as +legacy-only so the copier would retry it, and the copier's own size arm deleted it. Every +oversized value that can exist on any node came through a local caller of that method during +the bridge period, and through nothing else. + +**This release removes the bridge, so it removes the hole.** There is one store; its `put` +refuses anything over the ceiling, and refuses *before* it writes, so there is no partial state +for a later pass to find and no key recorded anywhere. There is no `LmdbStorage::put` left to +take an unbounded value in the first place. The read path and the repair path refuse over the +ceiling too, so a file planted by hand is refused rather than served. + +**Nothing is preserved and no sidecar is built.** An earlier draft added one, on the reasoning +that no peer can hold a copy of an over-ceiling value and no repair can fetch one, so deleting +it destroys the only copy. Both halves of that are true and the conclusion still does not +follow: there is no valid chunk there to be the only copy *of*. Building somewhere to keep +invalid values would be building for a case this release makes unreachable, and the cost of +doing it was measured — the sidecar was written, and adversarial review found two real defects +inside it, in code that existed only to serve a case that cannot arise. It was cut before it +shipped, and it is not coming back here. + +A legacy directory that happens to contain such a value is kept if it is unmarked and removed if +it carries the mark, exactly like every other directory, and for exactly the same reasons. This +release never looks inside one, so its contents are not a factor in either verdict. + +### The penalty is restored by the release after this one, not by this one + +The original plan had this release delete the old store and restore the close-group storage +penalty together. They are now separated, and the separation is the point. + +The upgrade monitor picks the newest eligible release rather than the next one. A node that was +offline while the migration ran therefore arrives here having never migrated, holding a legacy +store this build cannot read. This release keeps that store rather than deleting it, which is +right for its data, but the node cannot serve those chunks and its close group will notice. +Restoring the accusation in the same release would slash that node for a state it had no chance +to leave, in the release that put it there. + +So this release ships with the penalty still held off, and the release after it restores the +penalty once the fleet has been observed clean for long enough to include the nodes that were +away. That is a one-line change to a build constant, and it costs one extra release to stop the +cleanup and the accusation landing on a stranded node at the same moment. + +What this leaves is one migration-era constant still in the tree after the release that was +meant to remove them all. That is a deliberate trade: the objective this protects is that no +release breaks the fleet, and it outranks the objective that no migration code survives. + +## Consequences + +### Positive + +- One store, one name, and about 5,600 lines of bridge and driver gone. +- The unheld-chunk penalty is still held off, and the release after this one restores it. +- **The cleanup never vetoes a start.** Not the same as "every node starts": the file store + is built first and one that cannot open still stops the node, exactly as before. What is + gone is the other reason, the refusal this release's first draft introduced. +- The per-volume migration lock and its deployment settings go with the migration. + +### Negative / Trade-offs + +- **A node that never finished migrating keeps its old store and does not get that disk + back.** That population is the short-of-disk nodes, and how large it is remains the open + fleet question ADR-0014 records — which is why that release now reports it on the wire, and + why this one is not published until the count is clean. Such a node runs and serves what it + migrated; what it does not do is reclaim the space, which it could not do safely under any + of the three answers considered here. +- **The two halves have very different emergency levers, which is why they are not in the same + release.** Restoring the penalty is a switch, undoable in minutes and from the fleet side. + Deleting the bridge can only be undone by not shipping the release at all — and not even by + rolling the binary back, since the upgrade monitor takes the node forward again. That + asymmetry is the reason the penalty waits for the release after this one rather than riding + along with the deletion, and it is a decision rather than a scheduling accident. +- `ChunkStore` and its module were renamed from `FileStore` and `file_store.rs`. Callers of + the facade did not change, because they already used that name — but **`FileStore`, + `FileStoreConfig`, `StoreLayout`, `VerifyReport` and `LEGACY_ENV_DIR` were exported too**, + and all five are gone from the public API along with the LMDB and migration types. A + downstream crate importing any of them stops compiling, so they belong in the release + notes and not only in this list. Two caveats on how far that narrowing goes. Under the + `test-utils` feature `chunk_store` is a public module, so everything `pub` inside it — + `StoreLayout`, `CapacityVerdict`, every inherent method — is still importable; the narrowing + above is the default build's surface. And `ChunkStore` keeps its public inherent methods, + which is a wider surface than the re-export list suggests on its own. + +### Neutral / Operational + +- `ANT_SUSPEND_UNHELD_CHUNK_PENALTY` still works and still logs loudly when it disagrees + with the build. +- `storage.migration` and `storage.db_size_gb` are gone from the configuration. The second + capped a memory map that no longer exists, and a setting that silently does nothing is + worse than one that is absent. Nothing declares `deny_unknown_fields`, so a config file + written by the previous release still loads with both keys in it, which is what stops + every node on the fleet failing to start at once on upgrade. There is a test for that, + because adding that attribute later would look harmless. Worth knowing what that test is: a + hand-written fragment carrying the two removed keys, not a complete file emitted by the + previous release. It proves those two keys are tolerated; it does not prove a whole persisted + config round-trips, and an incompatibility in a field it omits would not be caught by it. +- There is no supported way to make this build read an old chunk store, and no way to stop + it clearing one up. A leftover it declines to touch — a link, an unmarked store with chunks + in it, or one whose state cannot be established — is named in a warning carrying + `migration_event = "legacy_store_left"`, and is the operator's to remove. + +## Validation + +**Proved here.** A marked environment is removed and its disk comes back; so is a marked +tombstone, and so is an empty one. An **unmarked** environment with chunks in it is still +there afterwards, and so is an unmarked tombstone — those two are the data-loss tests, and +they are the ones that would have gone red against the draft that deleted everything. A node +whose root does not exist yet is fine. Four directories that only *look* like leftovers +(`chunks.mdb.retired-keep-this`, `chunks.mdb.backup`, `.65`, `.007`) are all still there, and +the name matcher is asserted directly on both sides. A linked environment is untouched: the +link is still there and what it points at still has its data, which is the pair of mistakes +prefix matching and link following would each have made. + +A directory called `RETIRED` sitting where the mark would be does not authorise anything, and +the store it is in is still there afterwards. The deletion's order is asserted directly: with +a subdirectory made unremovable, the attempt fails and the mark has to still be there, because +that is what lets the next start recognise the directory rather than treat it as unmigrated +forever. + +A value over the ceiling is refused by `put`, and refused *before* anything is written: the +store does not claim it, no file is left under its name, and a restart onto the same directory +finds nothing to index. Addressed to its own bytes on purpose, so the refusal is the size arm +and not the content-address arm — checking the wrong arm would pass while the ceiling was gone. +Mutation-checked: with the size branch deleted the test fails, which is what says it is testing +the ceiling rather than testing that something went wrong. + +Three further properties are pinned that the earlier draft had no test for, because it had no +code for them either. + +**The correspondence with the fleet signal.** A root is staged carrying at least one shape for +every verdict the classifier can return — harmless three ways (a marked live environment, a +marked tombstone, an empty one), holding two (an unmarked tombstone with chunks in it, a link +wearing a tombstone's name), unreadable two (something that is not the mark using the mark's +name, and a plain file wearing a tombstone's name) — plus the near misses that are not in the +name set at all. Each is classified before anything is removed, and afterwards each is asserted +gone exactly when it was called harmless and still there exactly when it was not. Both +directions, so a classifier that drifts either way fails here; mutation-checked by making a link +classify harmless, which the test catches. + +**A marked directory is removed even when nothing was copied into this node's file store**, +staged with no file store at all. That is a shedding node's ordinary state, and pinning it is +what stops somebody later adding a local-copy check that would strand every such directory for +ever. + +**Sixty-six leftovers**, the whole namespace this release accepts, are removed by one start. And +**a root that cannot be listed** has nothing removed from it, which is the case that used to +return silently and left this record promising a warning nothing emitted. + +**What these do NOT prove, said rather than implied.** The sixty-six-leftover test asserts that +all of them go; it does not observe how many threads did it, so it would still pass if the one +sequential worker went back to one per directory. The deletion ORDER — mark last — has its +failure half asserted only on Unix, because staging a part-way failure needs a permission bit +there and an open handle on Windows; off Unix that test proves only that a successful deletion +leaves nothing behind. And the entry-level unreadable case is not staged: the test that makes a +root unlistable exercises `read_dir` itself failing, not one `DirEntry` failing inside a +readable root, so turning that arm from a refusal into a skip would restore the false-green and +stay green here. + +Through `NodeBuilder::build()`: a node with an unmigrated store starts under both +`storage.enabled = true` and `false` and still has its chunks afterwards; a node with a marked +one starts and the leftover goes; and a node whose file store cannot open — staged with a file +where the chunk directory has to be — fails to build with its old store still intact, which is +what pins the deletion behind the replacement. The penalty staying SUSPENDED is pinned by a test +on the shipped constant, which fails if it is flipped — in either direction. The existing tests +set the switch both ways on purpose, so none of them could ever notice which way it was +compiled, which is how a suspension outlives the thing it was suspended for. + +**Deleted, and what replaced it.** ADR-0014's validation section describes four harnesses. +Most of three of them existed to prove the bridge worked: that the disk came back when the +old store was deleted, that a node killed mid-copy lost nothing, and that several nodes on +one disk took turns. There is no bridge left for those to test. The fourth, which measures +what one file per chunk costs at scale, stays. + +Not all of it went, and saying it did was wrong. Two tests inside the crash harness were +never about the bridge: that a process killed mid-publish leaves no chunk the store cannot +serve, and that what an interrupted write leaves behind is swept. Those are about the store's +own publish path, which is now the only one there is, so they matter more after this release +rather than less. They are back as `tests/chunk_store_crash_safety.rs` and run in CI. A third +property, that engine shutdown waits for a detached store write, is named under the gaps +below. + +**The hardening release's two tests, one deleted and one that must keep passing.** The release +that hardened the migration added exactly two, and this release does something different with +each, so both are named rather than left to be noticed in a diff. +`a_store_left_midway_by_the_previous_release_keeps_its_place` proved that an upgrade picks a +half-finished migration up where it left off instead of restarting its clock — it drives +`copy_batch`, `migration_phase`, `legacy_only_keys` and `migration_state`, all of which this +release deletes. It goes with its subject: there is no migration left for an upgrade to +continue, and a test of one cannot be rewritten against a release that has none. +`retiring_keeps_a_pinned_root_answerable_and_clearing_does_not` is the opposite case. It is why +the commitment rotation has no emptiness branch, it touches nothing this release removes, and it +still passes here. **It must go on passing**, because the branch it rules out is exactly the one +an earlier draft of this release was going to reinstate. + +That leaves the loopback filesystem job with nothing to run, and deleting it would quietly +drop ext4, XFS and btrfs coverage of the store itself. It now runs the storage unit tests +against each mounted filesystem instead, which is what still has something to say there: +publishing through a temporary and a rename, flushing, deleting, and rebuilding an index +from the names. + +**Not proved here, and inherited from ADR-0014.** Forced power loss on the five filesystems. +Scale at one and ten million keys. How many short-of-disk nodes can clear the possession +gate. Under ADR-0014 such a node keeps serving from both stores; under this one it starts, +serves what it migrated, keeps what it did not, and never gets that disk back. + +**Coverage this release drops, named rather than lost.** A harness proved that +`ReplicationEngine::shutdown()` waits for a store write whose awaiter was dropped before it +returns. It was written against the old store and went with it. The property is still +current and still claimed by that method's own documentation, and nothing tests it now. It +needs a live P2P node to stage, which is why it is called out here rather than quietly +rewritten in the same change that deleted it. + +**A fleet gate this decision adds.** Before this ships, the fleet has to show that nodes are +actually on the file store, because this is the release that stops any of them going back. +ADR-0014 puts the answer on the wire: every node announces `migration/legacy`, +`migration/files` or `migration/unknown` in the user agent it already sends with every signed +message, and each node counts what it sees around it. The gate is that nothing reports +`legacy` or `unknown` for itself, and that no observed peer reports `legacy`, `unknown`, or +nothing at all, over several consecutive days, against a roster of nodes we expect to hear +from. A peer that reports nothing is running a build from before the signal existed, which is +not evidence of anything having finished; the tally counts it as outstanding for that reason. Silence is not readiness: +a zero from a collector that scraped nothing looks exactly like a zero from a clean fleet, +which is why the count needs a denominator and not just a numerator. + +That gate cannot be made airtight, and it is worth being honest about which part is soft. +Nodes that are offline for the whole window are in nobody's count and come back afterwards; +that is the population this decision knowingly cleans up rather than the one it waits for. + +**Where that gate stands as this is written, which is nowhere yet.** The signal first shipped in +`v0.19.0-beta.1`, published 2026-09-09, so there is under a day of it against a gate that asks +for several consecutive days. More to the point, the beta's own lines do not mean yet what the +gate needs them to mean: the staged migrations have not started, so a node reporting `files` +today is reporting that it has nothing to move rather than that it has finished moving it, and +those two are indistinguishable in the tally. And the population the gate exists for — the +community nodes, and the NTFS hosts where the store's case-folding and directory-entry +behaviour differ from ours — is not reporting to us at all. **No count taken before the staged +migrations run should be read as progress towards this gate**, and this release is not published +on the strength of one. + +## What this release does not fix + +Named rather than implied. None is a regression; each is the state before this change. + +- **A node that kept an unreadable store does not get that disk back**, and being short of disk + is how that costs it: it fails to take on chunks it is responsible for. Note that this + release does not add the unheld-chunk accusation on top — that lane is still suspended here + and returns in the release after. What such a node is exposed to is the commitment-bound + lane, which every release enforces. The remedy is the operator's, and the warning names the + directory. +- **Neither build re-marks or retries the way retirement did.** That release would retry a + failed final removal many times and write the mark back if it could not finish. This one makes + one attempt per start and leaves the rest to the next start, which is a longer wait on a host + where something holds the directory open — antivirus on Windows is the realistic case. What it + leaves behind is an empty unmarked directory, which the next start reads as harmless and + removes, so the cost is delay rather than a stuck node. +- **Off Unix the path check and the unlink are not one operation.** They are on Unix, where the + deletion is made against a handle opened `O_NOFOLLOW`; `std` offers no portable `unlinkat`, so + the other platforms keep the path-based deleter and the window the previous release's deleter + also had. +- **On a case-folding filesystem a leftover can be kept that should have gone.** Tombstone names + are matched exactly, as strings, against what `read_dir` reports, while NTFS and a + default-configured APFS compare names case-insensitively. A directory stored as + `CHUNKS.MDB.RETIRED` is the same file to the filesystem and a different string to the matcher, + so it is not recognised and is kept for ever. That is the safe direction — the failure is disk + not returned, never data removed — and it takes an operator having renamed something, since + retirement only ever writes lower case. The live name is unaffected: it is looked up by name + rather than matched from a listing, so the filesystem's own comparison finds it. +- **`deploy/terraform/cloud-init/worker.yml` still cannot start a node**: it passes no rewards + address, which production mode requires. The binary also sits in `/usr/local/bin` while the + unit runs under `ProtectSystem=strict`, so an in-process upgrade cannot replace it. Both + predate this work and belong to whoever owns that deployment. That path is not evidence for + the fleet gate until they are fixed. +- **Off Unix a chunk is published under its final name**, so a power loss can leave a real + chunk name over partial bytes, and a commitment built before anything reads it claims a + chunk the node cannot produce. ADR-0014 states this; the forced power-loss run is still an + open gate. +- **There is no supported way to hold a node on an earlier release.** This is named here + because it is what makes the fleet gate a gate rather than a preference. The on-disk format + rolls back cleanly — the previous release reads the same one-file-per-chunk layout, and a + legacy directory this release kept is one it can still pick up and finish — but the + *operation* does not: `build_upgrade_monitor` is called unconditionally, `UpgradeConfig` has + no field that disables it, and the monitor takes the newest eligible release, so a node put + back on the previous binary is dragged forward again within the hour. Rolling back to + v0.18.1 is worse than unsupported, because chunks this release accepted exist only in the + file store that build does not have. + + The fix is an upgrade-subsystem one — a persisted disable, or a version ceiling — and it is + deliberately not in this release: it changes the mechanism every node uses to take every + release, which is not a change to make in the release that also deletes a store. It belongs + in its own change, with its own evidence. + + What must not be said here, and an earlier draft of this record did say it, is that no state + this release creates would ever have wanted a rollback. A marked directory can hold shed + chunks, which are bytes the previous release proved the close group holds and deliberately did + not copy locally. Deleting them is right and is what that release was about to do — but it is + irreversible on this node, and rolling back would not bring them back either, because the + previous release would have deleted them too. The remedy for those keys was never local: it is + replication fetching them from the peers that proved they hold them. + +## Notes for AI-assisted work + +Drafted with AI assistance. Not to be marked Accepted without human review. diff --git a/docs/adr/README.md b/docs/adr/README.md index 5061984e..f4585acd 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -36,3 +36,4 @@ See [`TOOLING.md`](./TOOLING.md) for `adrs`, `adr-kit`, and AI harness setup. - [ADR-0013: Settlement version and pre-payment compatibility](./ADR-0013-settlement-version-and-pre-payment-compatibility.md) - [ADR-0015: Direct browser clients over WebRTC Direct](./ADR-0015-direct-browser-clients-over-webrtc-direct.md) - [ADR-0016: Pointers — paid mutable references with an immutable owner](./ADR-0016-pointers-immutable-owner.md) +- [ADR-0022: Remove the LMDB Chunk Store](./ADR-0022-remove-the-lmdb-chunk-store.md) diff --git a/src/config.rs b/src/config.rs index 09b06144..7e7cc447 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,6 +1,5 @@ //! Configuration for ant-node. -use crate::storage::MigrationConfig; use evmlib::Network as EvmNetwork; use serde::{Deserialize, Serialize}; use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4}; @@ -550,7 +549,7 @@ const fn default_staged_rollout_hours() -> u64 { /// Controls how chunks are stored, including: /// - Whether storage is enabled /// - Content verification on read -/// - Database size limits (auto-scales with available disk by default) +/// - How much free disk to leave unused #[derive(Debug, Clone, Serialize, Deserialize)] pub struct StorageConfig { /// Enable chunk storage. @@ -563,27 +562,12 @@ pub struct StorageConfig { #[serde(default = "default_storage_verify_on_read")] pub verify_on_read: bool, - /// Explicit LMDB database size cap in GiB. - /// - /// When set to 0 (default), the map size is computed automatically from - /// available disk space at startup and grows on demand when the operator - /// adds storage. Set a non-zero value to impose a hard cap. - #[serde(default)] - pub db_size_gb: usize, - /// Minimum free disk space (in MiB) to preserve on the storage partition. /// /// Writes are refused when available space drops below this threshold, /// preventing the node from filling the disk completely. Default: 500 MiB. #[serde(default = "default_disk_reserve_mb")] pub disk_reserve_mb: u64, - - /// Controls for moving this node off the legacy LMDB chunk store. - /// - /// The two release switches inside it are deliberately not serialised: see - /// [`MigrationConfig`]. - #[serde(default)] - pub migration: MigrationConfig, } impl Default for StorageConfig { @@ -591,14 +575,12 @@ impl Default for StorageConfig { Self { enabled: default_storage_enabled(), verify_on_read: default_storage_verify_on_read(), - db_size_gb: 0, disk_reserve_mb: default_disk_reserve_mb(), - migration: MigrationConfig::default(), } } } -/// Default: 500 MiB — matches `DEFAULT_DISK_RESERVE` in `storage::lmdb`. +/// Default: 500 MiB — matches `DEFAULT_DISK_RESERVE` in `storage`. const fn default_disk_reserve_mb() -> u64 { 500 } @@ -748,32 +730,59 @@ fn default_testnet_bootstrap() -> Vec { #[allow(clippy::unwrap_used, clippy::expect_used)] mod tests { - use super::*; - use serial_test::serial; - + /// A config file written by the previous release still loads. + /// + /// The settings that drove the migration off the old chunk store are gone from this + /// build, and so is the database size cap, which configured a memory map that no longer + /// exists. Every node on the fleet has a config file on disk carrying them, written by + /// the release that did the migrating. If those keys made the file unparseable, every + /// one of those nodes would fail to start on upgrade, all at once. + /// + /// Nothing declares `deny_unknown_fields`, so serde ignores them. That is the behaviour + /// this depends on, which makes it worth a test rather than an assumption: adding that + /// attribute later would look harmless and would take down the fleet. #[test] - fn the_shipped_storage_config_parses() { - // The migration section is operator-facing, so a typo in it would only surface on - // a node that had already shipped. Only `[storage]` is checked: the rest of - // `production.toml` does not currently deserialize as a `NodeConfig` (its - // `evm_network` is a bare string where an internally tagged enum is expected), - // which is a separate, pre-existing problem. - let raw = include_str!("../config/production.toml"); - let doc: toml::Value = toml::from_str(raw).expect("production.toml must be valid TOML"); - let storage = doc.get("storage").expect("a [storage] section").clone(); - let config: StorageConfig = storage.try_into().expect("[storage] must deserialize"); - - assert!(config.migration.enabled); - assert!(config.migration.dual_write_legacy); - assert_eq!(config.migration.shed_hold_hours, 72); - assert_eq!(config.migration.copier_throttle_mib_per_sec, 32); - assert_eq!(config.migration.copier_slack_mb, 2048); - // The release switches are absent from the file on purpose, so they come from the - // build rather than from whatever an operator's config last recorded. - let build = MigrationConfig::default(); - assert_eq!(config.migration.retire_legacy, build.retire_legacy); + fn a_config_file_from_the_previous_release_still_loads() { + let previous = r#" +[network] +port = 10000 + +[storage] +enabled = true +verify_on_read = true +db_size_gb = 32 +disk_reserve_mb = 500 + +[storage.migration] +shed_hold_hours = 72 +wave_hours = 24 +copier_throttle_mib_per_sec = 32 +copier_slack_mb = 2048 +retire_delay_hours = 4 + +[payment] +rewards_address = "0x0000000000000000000000000000000000000001" +"#; + let dir = tempfile::TempDir::new().expect("temp dir"); + let path = dir.path().join("config.toml"); + std::fs::write(&path, previous).expect("write the previous release's config"); + + // Through the loader a node actually uses, not a hand-picked table. The whole file + // has to parse, because that is what a node does with it on start. + let parsed = NodeConfig::from_file(&path) + .expect("a config file from the previous release must still load"); + + assert!(parsed.storage.enabled); + assert!(parsed.storage.verify_on_read); + assert_eq!( + parsed.storage.disk_reserve_mb, 500, + "the settings this build still uses must survive the ones it dropped" + ); } + use super::*; + use serial_test::serial; + #[test] fn test_default_config_has_cache_capacity() { let config = PaymentConfig::default(); diff --git a/src/devnet.rs b/src/devnet.rs index b6f75bed..649679c1 100644 --- a/src/devnet.rs +++ b/src/devnet.rs @@ -944,7 +944,7 @@ impl Devnet { }; let storage = ChunkStore::new(storage_config) .await - .map_err(|e| DevnetError::Core(format!("Failed to create LMDB storage: {e}")))?; + .map_err(|e| DevnetError::Core(format!("Failed to create the chunk store: {e}")))?; let evm_config = EvmVerifierConfig { network: config diff --git a/src/lib.rs b/src/lib.rs index 852c837a..e589cae6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -84,7 +84,7 @@ pub use pointer::{ Pointer, PointerState, PointerStore, PointerTarget, PointerTargetKind, PutOutcome, }; pub use replication::{config::ReplicationConfig, ReplicationEngine}; -pub use storage::{AntProtocol, ChunkStore, ChunkStoreConfig, LmdbStorage, LmdbStorageConfig}; +pub use storage::{AntProtocol, ChunkStore, ChunkStoreConfig}; /// Re-exports from `saorsa-core` so downstream crates (e.g. `ant-client`) /// can depend on `ant-node` alone without a direct `saorsa-core` dependency. diff --git a/src/node.rs b/src/node.rs index 56813244..238724ea 100644 --- a/src/node.rs +++ b/src/node.rs @@ -40,14 +40,6 @@ use tokio_util::task::TaskTracker; #[cfg(unix)] use tokio::signal::unix::{signal, SignalKind}; -/// How long shutdown waits for the storage migration to reach a stopping point. -/// -/// Generous, because interrupting a copy mid-chunk costs nothing (every step is -/// idempotent and re-derived at the next start) but interrupting the drain that precedes -/// removing the legacy store is worth avoiding. Bounded, because a step that will not -/// finish must not hold the process open. -const MIGRATION_SHUTDOWN_GRACE: std::time::Duration = std::time::Duration::from_secs(30); - /// How long shutdown waits for in-flight request handlers to finish. /// /// Short, because these are single request/response exchanges and the peer will retry. @@ -125,10 +117,11 @@ impl NodeBuilder { // Ensure root directory exists std::fs::create_dir_all(&self.config.root_dir)?; - // One release-level decision, applied before anything can audit: while the fleet - // moves off the legacy chunk store, a peer is not penalised for failing to hold a - // chunk it was supposed to be holding. It is still penalised for failing a - // commitment-bound audit. Audits of both kinds run and record throughout. + // One release-level decision, applied before anything can audit. It stays suspended + // here: this release deletes the old chunk store, and a node that was away while the + // migration ran arrives holding one it cannot read, so restoring the accusation now + // would slash it in the release that stranded it. The release after this one restores + // it. The commitment-bound audit has penalised throughout and still does. crate::replication::config::apply_close_group_storage_penalty_policy(); // Create shutdown token @@ -171,6 +164,35 @@ impl NodeBuilder { (None, None) }; + // Only now, and only if a store was actually opened. Clearing up after the storage + // migration deletes directories the previous release had finished with, and + // "finished with" means their chunks are in the file store — which is only true if + // the file store is there. Doing this earlier put the deletion in front of a + // constructor that can still fail on an unreadable layout, a directory it cannot + // create, or a lock another process has not let go of, and a node that lost both + // stores that way had nothing to go back to. + // + // A node with storage switched off never builds one at all, so it never establishes + // that anything was copied anywhere, and it does not delete. It also has no use for + // the disk it would recover. Leaving the directory costs space on a node that is not + // storing anything anyway, and keeps its contents recoverable by the release that + // can read them. + // + // Waiting costs nothing in what this node reports. Its user agent was fixed when the + // transport was built a moment ago, but everything removed here is a leftover the + // signal already reads as finished with — carrying the mark or being empty is both + // what makes it removable and what makes it harmless — so the node announces `files` + // whether the deletion has finished, is still running, or has not started. + if ant_protocol.is_some() { + crate::storage::legacy_artifacts::clean_up(&self.config.root_dir); + } else { + info!( + "Chunk storage is disabled, so anything the storage migration left behind is \ + being left where it is: nothing here can establish that its chunks were \ + copied anywhere." + ); + } + let p2p_arc = Arc::new(p2p_node); // Wire the P2PNode handle into AntProtocol so payment proofs can query @@ -179,7 +201,7 @@ impl NodeBuilder { protocol.attach_p2p_node(Arc::clone(&p2p_arc)); } - let (replication_engine, migration_task) = match (&ant_protocol, fresh_write_rx) { + let replication_engine = match (&ant_protocol, fresh_write_rx) { (Some(protocol), Some(fresh_rx)) => { Self::build_replication_engine( protocol, @@ -192,7 +214,7 @@ impl NodeBuilder { ) .await? } - _ => (None, None), + _ => None, }; let node = RunningNode { @@ -205,7 +227,6 @@ impl NodeBuilder { ant_protocol, replication_engine, protocol_task: None, - migration_task, protocol_children: TaskTracker::new(), #[cfg(feature = "webrtc-direct")] webrtc_direct_task: None, @@ -217,18 +238,13 @@ impl NodeBuilder { Ok(node) } - /// Start the replication engine and, if this node still has one, the migration off - /// the legacy chunk store. - /// - /// The two are built together because the migration cannot run without the engine: - /// it needs the commitment state, which holds the veto on deleting the old store, - /// and the live routing view that says which chunks this node must never give up. + /// Start the replication engine. /// /// # Errors /// - /// Returns an error only when the engine fails to start on a node that has a legacy - /// store to migrate. On a node with nothing to migrate an engine failure is logged - /// and the node runs without one, as it always has. + /// Never, currently: an engine that fails to start is logged and the node runs without + /// one, as it always has. The signature keeps its `Result` because the caller's does, + /// and because the migration release did have a case that had to refuse. async fn build_replication_engine( protocol: &Arc, repl_config: ReplicationConfig, @@ -237,7 +253,7 @@ impl NodeBuilder { root_dir: &Path, fresh_rx: UnboundedReceiver, shutdown: &CancellationToken, - ) -> Result<(Option, Option>)> { + ) -> Result> { let mut engine = match ReplicationEngine::new( repl_config, Arc::clone(p2p), @@ -252,22 +268,8 @@ impl NodeBuilder { { Ok(engine) => engine, Err(e) => { - // A node that still has a legacy chunk store depends on this engine for - // the commitment state, the routing view and the possession challenges - // the migration cannot proceed without. Carrying on would leave it - // serving from both stores forever, never reclaiming its disk, which is - // the condition this release exists to end. Refuse to start instead of - // running in it indefinitely. - if protocol.storage().has_legacy() { - return Err(Error::Startup(format!( - "This node has a legacy chunk store to migrate but the \ - replication engine did not start: {e}. Without it the \ - migration cannot run and the disk is never reclaimed. \ - Fix the cause rather than running on." - ))); - } warn!("Failed to initialize replication engine: {e}"); - return Ok((None, None)); + return Ok(None); } }; @@ -297,10 +299,7 @@ impl NodeBuilder { .payment_verifier_arc() .attach_monetized_pin_sender(engine.monetized_pin_sender()); - let migration_task = - Self::spawn_storage_migration(protocol.storage(), p2p, &engine, shutdown.clone()); - - Ok((Some(engine), migration_task)) + Ok(Some(engine)) } /// Build the saorsa-core `NodeConfig` from our config. @@ -491,58 +490,23 @@ impl NodeBuilder { monitor } - - /// Start moving this node off the legacy LMDB chunk store, if it still has one. - /// - /// Started after the replication engine rather than with the store, because the - /// copier needs two things only the engine has: the commitment state, which owns the - /// retention veto on deleting the old store, and live routing, which is how the node - /// knows which chunks it is among the closest to and therefore must never give up. - fn spawn_storage_migration( - store: Arc, - p2p: &Arc, - engine: &ReplicationEngine, - shutdown: CancellationToken, - ) -> Option> { - if !crate::storage::migration::should_migrate(&store) { - return None; - } - let context = crate::storage::migration::MigrationContext { - p2p: Some(Arc::clone(p2p)), - self_id: Some(*p2p.peer_id()), - self_xor: crate::client::peer_id_to_xor_name(&p2p.peer_id().to_string()), - commitment: Some(Arc::clone(engine.commitment_state())), - replication: Some(Arc::clone(engine.config())), - sync_state: Some(Arc::clone(engine.sync_state())), - audit_challenge_coordinator: Some(Arc::clone(engine.audit_challenge_coordinator())), - peer_commitments: Some(Arc::clone(engine.last_commitment_by_peer())), - close_group_size: engine.config().close_group_size, - }; - Some(tokio::spawn(async move { - crate::storage::migration::run(store, context, shutdown).await; - })) - } - /// Build the ANT protocol handler from config. /// - /// Initializes LMDB storage, payment verifier, and quote generator. + /// Initializes the chunk store, payment verifier, and quote generator. /// Wires ML-DSA-65 signing from the node's identity into the quote generator. async fn build_ant_protocol( config: &NodeConfig, identity: &NodeIdentity, close_group_size: usize, ) -> Result { - // Create LMDB storage let storage_config = ChunkStoreConfig { root_dir: config.root_dir.clone(), verify_on_read: config.storage.verify_on_read, - max_map_size: config.storage.db_size_gb.saturating_mul(1024 * 1024 * 1024), disk_reserve: config.storage.disk_reserve_mb.saturating_mul(MIB), - migration: config.storage.migration.clone(), }; let storage = ChunkStore::new(storage_config) .await - .map_err(|e| Error::Startup(format!("Failed to create LMDB storage: {e}")))?; + .map_err(|e| Error::Startup(format!("Failed to create the chunk store: {e}")))?; // Parse rewards address (required — node must know where to receive payments) let rewards_address = match config.payment.rewards_address { @@ -622,17 +586,12 @@ pub struct RunningNode { replication_engine: Option, /// Protocol message routing background task. protocol_task: Option>, - /// The task moving this node off the legacy chunk store, if it has one. - /// - /// Awaited before the replication engine and the P2P layer are torn down, because it - /// holds handles to both and is in the middle of reading and writing the chunk store. - migration_task: Option>, /// The per-message handler tasks the protocol loop spawns. /// /// Tracked rather than detached so shutdown can stop accepting work and then wait for /// what is already in flight. Aborting only the loop leaves its children running, and - /// a chunk read that outlives the loop keeps the legacy store busy exactly while the - /// migration is trying to drain it. + /// a chunk read that outlives the loop keeps working against a store the shutdown is + /// about to tear down. protocol_children: TaskTracker, /// ADR-0015 browser listener task. #[cfg(feature = "webrtc-direct")] @@ -930,25 +889,10 @@ impl RunningNode { }); } - // A node that still has a legacy chunk store and no task moving it off one is the - // failure this cannot be allowed to have silently: the store opens, serves the - // union of both, and never frees a byte. It happened once, during a rebase that - // dropped the spawn, and nothing noticed because a node without a legacy store - // starts no migration and every test built the store directly. Say so loudly. - if let Some(ref protocol) = self.ant_protocol { - if protocol.storage().has_legacy() && self.migration_task.is_none() { - error!( - migration_event = "not_started", - "This node still has a legacy chunk store but nothing is migrating it. \ - Its disk will never be reclaimed. This is a wiring fault, not a \ - configuration one: report it rather than working around it." - ); - } - } - info!("Node running, waiting for shutdown signal"); - // Run the main event loop with signal handling + // The main event loop, with signal handling. Everything above this starts + // something; this is where the node waits. self.run_event_loop().await } @@ -964,13 +908,20 @@ impl RunningNode { } } - // Protocol routing stops FIRST, loop and children both. The migration's last step - // drains the legacy store's in-flight reads, and inbound protocol traffic keeps - // starting new ones, so waiting on the migration while still serving requests can - // keep that drain from ever completing and hang shutdown. Aborting the accept loop - // alone would not do it: the requests already in flight run in their own tasks. + // Protocol routing stops first, loop and children both. The routing loop waits on + // `events.recv()` and has no cancellation branch of its own, and it holds an `Arc` + // on the P2P node that keeps the sender it is waiting on alive, so nothing else + // here will ever wake it. Left running it holds the chunk store and its + // single-process lock open after the node has returned. Aborting the accept loop + // alone is not enough either: the requests already in flight run in their own + // tasks, which is what the drain below is for. if let Some(handle) = self.protocol_task.take() { handle.abort(); + // Awaited, not just asked to stop. `abort` schedules cancellation; it does not + // establish that the task is gone, and what matters here is that it has + // dropped its `Arc` on the protocol and with it the store's single-process + // lock before this function returns. The join resolves as cancelled. + let _ = handle.await; } // Cancelled first, so anything still queued behind the concurrency permits gives // up rather than starting fresh storage work, then given a moment to finish what @@ -989,30 +940,6 @@ impl RunningNode { ); } - // Then the migration, awaited rather than aborted: it is mid-way through reading - // and writing the chunk store, and it holds the commitment state and the routing - // handle that the shutdown below is about to invalidate. It watches the same - // cancellation token, so this returns as soon as its current step does. Bounded, - // because a step that will not finish must not hold the process open. - if let Some(mut handle) = self.migration_task.take() { - // Awaited by reference, so a timeout leaves the handle here to abort rather - // than dropping it and letting the task run on detached through the engine and - // P2P teardown it depends on. - match tokio::time::timeout(MIGRATION_SHUTDOWN_GRACE, &mut handle).await { - Ok(Ok(())) => {} - Ok(Err(e)) => warn!("Storage migration task did not stop cleanly: {e}"), - Err(_) => { - warn!( - "Storage migration did not stop within {}s; stopping it. \ - Everything it does is idempotent and re-derived at the next start.", - MIGRATION_SHUTDOWN_GRACE.as_secs() - ); - handle.abort(); - let _ = handle.await; - } - } - } - // Shutdown replication engine before P2P so background tasks don't // use a dead P2P layer, and Arc references are released. if let Some(ref mut engine) = self.replication_engine { @@ -1246,6 +1173,8 @@ mod tests { use rand::Rng; use tempfile::TempDir; + use crate::storage::migration_signal::LEGACY_ENV_DIR; + /// The e2e port range, so a test bind never lands on a production or dev instance. const TEST_PORT_RANGE: std::ops::Range = 20000..60000; @@ -1255,79 +1184,6 @@ mod tests { /// A well-formed address that receives nothing; no chain is contacted in these tests. const TEST_REWARDS_ADDRESS: &str = "0x0000000000000000000000000000000000000001"; - /// A node with a legacy chunk store must get a migration task; one without must not. - /// - /// The spawn helper is tested directly because its *absence* is the failure mode that - /// already happened here: a rebase dropped the call, the store still opened and still - /// served, and no test could tell the difference. - #[tokio::test] - async fn a_legacy_store_gets_a_migration_task_and_a_fresh_node_does_not() { - let dir = TempDir::new().expect("temp dir"); - let root = dir.path().join("node"); - std::fs::create_dir_all(&root).expect("mkdir"); - - // Fresh node: nothing to migrate, so no task. - let fresh = Arc::new( - crate::storage::ChunkStore::new(crate::storage::ChunkStoreConfig { - root_dir: root.clone(), - ..crate::storage::ChunkStoreConfig::test_default() - }) - .await - .expect("open fresh"), - ); - assert!(!fresh.has_legacy()); - assert!( - !crate::storage::migration::should_migrate(&fresh), - "a node with no legacy store has nothing to migrate" - ); - drop(fresh); - - // Seed a legacy store, then reopen: now there is something to migrate. - { - let lmdb = crate::storage::LmdbStorage::new(crate::storage::LmdbStorageConfig { - root_dir: root.clone(), - verify_on_read: true, - max_map_size: 0, - disk_reserve: 0, - }) - .await - .expect("open legacy"); - let content = b"a chunk from before the migration"; - let addr = crate::client::compute_address(content); - lmdb.put(&addr, content).await.expect("put"); - lmdb.wait_idle().await; - } - let upgrading = Arc::new( - crate::storage::ChunkStore::new(crate::storage::ChunkStoreConfig { - root_dir: root.clone(), - ..crate::storage::ChunkStoreConfig::test_default() - }) - .await - .expect("open upgrading"), - ); - assert!(upgrading.has_legacy()); - assert!( - crate::storage::migration::should_migrate(&upgrading), - "a node with a legacy store must be migrated, or its disk is never reclaimed" - ); - } - - /// Seed a legacy LMDB store under `root` with one chunk, then close it. - async fn seed_legacy_store(root: &std::path::Path) { - let lmdb = crate::storage::LmdbStorage::new(crate::storage::LmdbStorageConfig { - root_dir: root.to_path_buf(), - verify_on_read: true, - max_map_size: 0, - disk_reserve: 0, - }) - .await - .expect("open legacy"); - let content = b"a chunk written before the migration"; - let addr = crate::client::compute_address(content); - lmdb.put(&addr, content).await.expect("put"); - lmdb.wait_idle().await; - } - /// A node config that builds without touching a chain or a real network. fn local_node_config(root: &std::path::Path, port: u16) -> NodeConfig { NodeConfig { @@ -1343,13 +1199,14 @@ mod tests { } } + /// A browser listener that cannot bind fails startup, and `run` still drains what it + /// had started before returning. #[cfg(feature = "webrtc-direct")] #[tokio::test] - async fn browser_startup_failure_drains_existing_migration() { + async fn browser_startup_failure_drains_before_returning() { let dir = TempDir::new().expect("temp dir"); let root = dir.path().join("node"); std::fs::create_dir_all(&root).expect("mkdir"); - seed_legacy_store(&root).await; let occupied = tokio::net::UdpSocket::bind("127.0.0.1:0") .await .expect("bind"); @@ -1357,36 +1214,22 @@ mod tests { config.webrtc_direct.enabled = true; config.webrtc_direct.bind = occupied.local_addr().expect("address"); let mut node = NodeBuilder::new(config).build().await.expect("build"); - assert!(node.migration_task.is_some()); let cancelled = node.shutdown.clone(); let result = node.run().await; assert!(result.is_err(), "occupied browser port must fail startup"); assert!(cancelled.is_cancelled()); - assert!( - node.migration_task.is_none(), - "migration must be joined before returning" - ); assert!(node.protocol_children.is_empty()); // The test runtime remains alive: cleanup must not rely on process exit. tokio::task::yield_now().await; } - /// A real, fully built node with a legacy store is actually migrating it. - /// - /// This goes through `build()` rather than calling the spawn helper, because the - /// failure that already happened here was the *call site* going missing, not the - /// helper being wrong. A test of the helper alone stays green through exactly that - /// bug. Deleting the spawn from `build()` must turn this red. + /// A node builds on a root with nothing left over from the old store. #[tokio::test] - async fn a_built_node_with_a_legacy_store_is_migrating_it() { + async fn a_node_builds_on_a_clean_root() { let dir = TempDir::new().expect("temp dir"); let root = dir.path().join("node"); std::fs::create_dir_all(&root).expect("mkdir"); - seed_legacy_store(&root).await; - // Ports are picked at random from the test range and a freshly released one can - // still be held for a moment, so a bind failure is retried rather than reported - // as a wiring fault. let mut built = None; let mut last_err = String::new(); for _ in 0..BIND_ATTEMPTS { @@ -1409,58 +1252,161 @@ mod tests { panic!("could not build a node after {BIND_ATTEMPTS} attempts: {last_err}"); }; - let storage_has_legacy = node - .ant_protocol - .as_ref() - .is_some_and(|p| p.storage().has_legacy()); - assert!( - storage_has_legacy, - "the node must have opened the legacy store this test seeded" - ); + node.shutdown.cancel(); + } + + /// A node with chunks in a store this build cannot read STARTS, however it is + /// configured, and does not lose them. + /// + /// This asserted the opposite until the release that shipped it was reconsidered twice. + /// First it refused to start, which serves nothing at all and cannot be recovered by + /// hand, because nothing can hold a node on an older build. Then it deleted whatever it + /// found, which is data loss for exactly the node that most needs the data: the upgrade + /// monitor picks the newest eligible release rather than the next one, so a node that was + /// offline through the previous release arrives here with everything it owns in that + /// directory and nothing in the file store. + /// + /// Both configurations, because they are different code paths: a node with + /// `storage.enabled = false` never builds a store, and the old environment is on its disk + /// just the same. + #[tokio::test] + async fn a_node_with_an_unmigrated_store_starts_and_keeps_it() { + for storage_enabled in [true, false] { + let dir = TempDir::new().expect("temp dir"); + let root = dir.path().join("node"); + let env = root.join(LEGACY_ENV_DIR); + std::fs::create_dir_all(&env).expect("mkdir"); + std::fs::write(env.join("data.mdb"), b"chunks that were never copied out") + .expect("seed"); + + let port = rand::thread_rng().gen_range(TEST_PORT_RANGE); + let mut config = local_node_config(&root, port); + config.storage.enabled = storage_enabled; + + let node = NodeBuilder::new(config).build().await.unwrap_or_else(|e| { + panic!( + "a node with an unmigrated store must start (storage.enabled = \ + {storage_enabled}): {e}" + ) + }); + settle(); + assert!( + env.join("data.mdb").exists(), + "chunks that were never copied out were deleted (storage.enabled = \ + {storage_enabled})" + ); + node.shutdown.cancel(); + } + } + + /// A node with storage switched off deletes nothing, even a leftover marked finished. + /// + /// It never builds a store, so nothing about this node establishes that those chunks were + /// copied anywhere: the mark is a claim made by a previous release about a file store + /// this process has not opened and will not open. It also has no use for the disk. So the + /// directory stays and stays recoverable. + #[tokio::test] + async fn a_node_with_storage_disabled_deletes_nothing() { + let dir = TempDir::new().expect("temp dir"); + let root = dir.path().join("node"); + let env = root.join(LEGACY_ENV_DIR); + std::fs::create_dir_all(&env).expect("mkdir"); + std::fs::write( + env.join("data.mdb"), + b"bytes retirement had already cleared", + ) + .expect("seed"); + std::fs::write(env.join("RETIRED"), b"retired").expect("mark"); + + let port = rand::thread_rng().gen_range(TEST_PORT_RANGE); + let mut config = local_node_config(&root, port); + config.storage.enabled = false; + let node = NodeBuilder::new(config).build().await.expect("must start"); + settle(); assert!( - node.migration_task.is_some(), - "a node holding a legacy chunk store came up with nothing migrating it, so \ - its disk would never be reclaimed" + env.join("data.mdb").exists(), + "a node that never opened a store deleted one on the strength of a mark it \ + could not check" ); + node.shutdown.cancel(); + } + + /// A store the migration finished with is removed, once the file store has opened. + #[tokio::test] + async fn a_finished_store_is_removed_once_the_replacement_opens() { + let dir = TempDir::new().expect("temp dir"); + let root = dir.path().join("node"); + let env = root.join(LEGACY_ENV_DIR); + std::fs::create_dir_all(&env).expect("mkdir"); + std::fs::write( + env.join("data.mdb"), + b"bytes retirement had already cleared", + ) + .expect("seed"); + std::fs::write(env.join("RETIRED"), b"retired").expect("mark"); + let port = rand::thread_rng().gen_range(TEST_PORT_RANGE); + let node = NodeBuilder::new(local_node_config(&root, port)) + .build() + .await + .expect("a node with a finished leftover must start"); + assert!(wait_gone(&env)); node.shutdown.cancel(); - if let Some(handle) = node.migration_task { - handle.abort(); - } } - /// A node with nothing to migrate does not start a driver for it. + /// Nothing is deleted until the store that replaced it has actually opened. + /// + /// "Finished with" means the chunks are in the file store, which is only true if the file + /// store opens. An earlier version deleted first and let the constructor fail behind it, + /// on an unreadable layout or a directory it could not create, and a node that lost both + /// stores that way had nothing left to go back to. #[tokio::test] - async fn a_built_node_without_a_legacy_store_starts_no_migration() { + async fn a_finished_store_survives_a_file_store_that_will_not_open() { let dir = TempDir::new().expect("temp dir"); let root = dir.path().join("node"); - std::fs::create_dir_all(&root).expect("mkdir"); + let env = root.join(LEGACY_ENV_DIR); + std::fs::create_dir_all(&env).expect("mkdir"); + std::fs::write( + env.join("data.mdb"), + b"bytes retirement had already cleared", + ) + .expect("seed"); + std::fs::write(env.join("RETIRED"), b"retired").expect("mark"); - let mut built = None; - let mut last_err = String::new(); - for _ in 0..BIND_ATTEMPTS { - let port = rand::thread_rng().gen_range(TEST_PORT_RANGE); - match NodeBuilder::new(local_node_config(&root, port)) - .build() - .await - { - Ok(node) => { - built = Some(node); - break; - } - Err(e) => { - last_err = e.to_string(); - tokio::time::sleep(std::time::Duration::from_millis(250)).await; - } + // A file where the chunk directory has to be, so the store cannot create it. + std::fs::write(root.join("chunks"), b"not a directory").expect("block the store"); + + let port = rand::thread_rng().gen_range(TEST_PORT_RANGE); + let built = NodeBuilder::new(local_node_config(&root, port)) + .build() + .await; + assert!( + built.is_err(), + "the file store was supposed to fail to open" + ); + settle(); + assert!( + env.join("data.mdb").exists(), + "the old store was deleted before the one replacing it could open" + ); + } + + /// The removal runs on its own thread. + fn settle() { + std::thread::sleep(std::time::Duration::from_millis(200)); + } + + /// The removal runs on its own thread, so give it a moment. + fn wait_gone(path: &std::path::Path) -> bool { + for _ in 0..200 { + if !path.exists() { + return true; } + std::thread::sleep(std::time::Duration::from_millis(10)); } - let Some(node) = built else { - panic!("could not build a node after {BIND_ATTEMPTS} attempts: {last_err}"); - }; - - assert!(node.migration_task.is_none()); - node.shutdown.cancel(); + false } + use super::*; use crate::config::NODES_SUBDIR; diff --git a/src/payment/metrics.rs b/src/payment/metrics.rs index b59c19f5..fe36002b 100644 --- a/src/payment/metrics.rs +++ b/src/payment/metrics.rs @@ -37,7 +37,7 @@ impl QuotingMetricsTracker { /// /// This is the deletion-aware path and the SINGLE source of truth for the /// priced record count: the handler calls it at quote time with the live - /// LMDB entry count (`current_chunks()`), so any record removed from + /// live chunk count (`current_chunks()`), so any record removed from /// storage — by delete, prune, or otherwise — is reflected on the next /// quote with no per-delete bookkeeping to keep in sync. `record_store` /// remains only an optimistic between-quote hint; the resync overwrites it. diff --git a/src/payment/verifier.rs b/src/payment/verifier.rs index 14925fdf..0d9a126d 100644 --- a/src/payment/verifier.rs +++ b/src/payment/verifier.rs @@ -682,7 +682,7 @@ pub struct PaymentVerifier { /// midpoint in the live DHT. `None` in unit tests that don't exercise /// live-DHT checks; production startup MUST call [`attach_p2p_node`]. p2p_node: RwLock>>, - /// LMDB storage handle, attached post-construction. Retained for + /// Chunk store handle, attached post-construction. Retained for /// store-backed verifier checks that need the authoritative on-disk record /// count without depending on a side counter that may drift from /// replication/repair/prune paths. NOTE: the ADR-0006 price floor does NOT diff --git a/src/pointer/service.rs b/src/pointer/service.rs index bdda6c54..9c5b0749 100644 --- a/src/pointer/service.rs +++ b/src/pointer/service.rs @@ -425,10 +425,8 @@ mod tests { let chunks = ChunkStore::new(crate::storage::ChunkStoreConfig { root_dir: dir.path().to_path_buf(), verify_on_read: false, - max_map_size: 0, // Larger than any disk, so every capacity question answers "full". disk_reserve: u64::MAX, - migration: crate::storage::MigrationConfig::default(), }) .await .expect("chunk store"); @@ -477,9 +475,7 @@ mod tests { let chunks = ChunkStore::new(crate::storage::ChunkStoreConfig { root_dir: dir.path().to_path_buf(), verify_on_read: false, - max_map_size: 0, disk_reserve: 0, - migration: crate::storage::MigrationConfig::default(), }) .await .expect("chunk store"); diff --git a/src/pointer/store.rs b/src/pointer/store.rs index 3628d173..50d42e66 100644 --- a/src/pointer/store.rs +++ b/src/pointer/store.rs @@ -1494,9 +1494,7 @@ mod tests { let chunks = crate::storage::ChunkStore::new(crate::storage::ChunkStoreConfig { root_dir: dir.path().to_path_buf(), verify_on_read: false, - max_map_size: 0, disk_reserve: 0, - migration: crate::storage::MigrationConfig::default(), }) .await .expect("chunk store"); @@ -1536,9 +1534,7 @@ mod tests { let chunks = crate::storage::ChunkStore::new(crate::storage::ChunkStoreConfig { root_dir: dir.path().to_path_buf(), verify_on_read: false, - max_map_size: 0, disk_reserve: 0, - migration: crate::storage::MigrationConfig::default(), }) .await .expect("chunk store"); diff --git a/src/replication/admission.rs b/src/replication/admission.rs index 445d5644..23669ffa 100644 --- a/src/replication/admission.rs +++ b/src/replication/admission.rs @@ -202,7 +202,7 @@ mod tests { // ----------------------------------------------------------------------- // AdmissionResult construction helpers for pure-logic tests // - // The full `admit_hints` function requires a live DHT + LMDB backend. + // The full `admit_hints` function requires a live DHT and chunk store. // For unit tests we directly exercise: // 1. Cross-set precedence logic // 2. Deduplication logic diff --git a/src/replication/audit.rs b/src/replication/audit.rs index 9b312bb3..867b8368 100644 --- a/src/replication/audit.rs +++ b/src/replication/audit.rs @@ -896,9 +896,7 @@ mod tests { let config = ChunkStoreConfig { root_dir: temp_dir.path().to_path_buf(), verify_on_read: false, - max_map_size: 0, disk_reserve: 0, - ..ChunkStoreConfig::test_default() }; let storage = ChunkStore::new(config).await.expect("create storage"); (storage, temp_dir) diff --git a/src/replication/config.rs b/src/replication/config.rs index 276ba1bb..0eb49093 100644 --- a/src/replication/config.rs +++ b/src/replication/config.rs @@ -230,7 +230,7 @@ pub const AUDIT_RESPONDER_TOP_ORIGINS: usize = 10; /// round-1 proofs from starving the light audits, and bounds concurrent /// multi-gigabyte hashing to this many at once. Two allows overlap without /// admitting many simultaneous full-subtree hashes; there is little benefit in -/// more concurrent large LMDB scans against one disk. +/// more concurrent large store scans against one disk. pub const MAX_CONCURRENT_SUBTREE_ROUND1: usize = 2; /// Per-peer concurrency cap for the heavy subtree-audit round 1. One in-flight @@ -293,7 +293,7 @@ pub const SUBTREE_ROUND1_WORK_BURST_BYTES: i64 = 8 * 1024 * 1024 * 1024; /// Floor charged against the round-1 work budget per leaf attempted, in bytes. /// /// The budget counts content bytes, which is the right unit for the hashing but -/// misses what a leaf costs before its size is known: an LMDB point lookup with +/// misses what a leaf costs before its size is known: a point lookup with /// its retries, and a `spawn_blocking` dispatch and join. Nothing bounds a /// chunk from below, so a commitment made of a million tiny records would run a /// full subtree of reads and task round-trips per audit while charging almost @@ -638,7 +638,7 @@ pub const MAX_VERIFICATION_KEYS_PER_CYCLE: usize = 8_192; /// /// Senders aggregate all keys for a peer into one request. Matching this limit /// to the cycle bound lets an honest round use one request per peer while still -/// bounding the LMDB work performed on the responder's serial replication +/// bounding the storage work performed on the responder's serial replication /// message path. Oversized requests are rejected as an empty, wire-compatible /// verification response. pub const MAX_INCOMING_VERIFICATION_KEYS: usize = MAX_VERIFICATION_KEYS_PER_CYCLE; @@ -700,10 +700,25 @@ pub(crate) const CAPACITY_BLOCKED_RETRY: Duration = /// Trust event weight for confirmed audit failures. pub const AUDIT_FAILURE_TRUST_WEIGHT: f64 = 5.0; -/// Whether this build penalises a peer for not holding a chunk it was supposed to hold. +/// Whether this build HOLDS OFF the penalty for not holding a chunk it was supposed to hold. /// -/// **`true` while the fleet moves off the legacy LMDB chunk store; back to `false` once it -/// has.** Flipping it is a one-line change in one release. +/// `true` suspends the penalty. The name says `SUSPEND`; read it that way, because a reader who +/// takes it as "does this build penalise" gets the answer backwards, which is how a comment +/// three lines down came to claim the opposite of what ships. +/// +/// **Still `true`, and deliberately not flipped by this release.** It was raised while the +/// fleet moved off the legacy LMDB chunk store, because a node that has to give up chunks +/// cannot stop its peers penalising it for that, so the peers had to stop first. +/// +/// Restoring it belongs in a release *after* this one, not in this one. The upgrade monitor +/// picks the newest eligible release rather than the next, so a node that was offline while +/// the migration ran arrives here having never migrated, holding a legacy store this build +/// cannot read. This release keeps that store rather than deleting it, which is the right +/// answer for its data, but the node cannot serve those chunks. Restoring the penalty in the +/// same release would slash exactly that node, for a state it had no chance to leave, in the +/// release that stranded it. Two changes, two releases: this one removes the old store, and +/// the next one restores the accusation once the fleet has been observed clean for long +/// enough to include the nodes that were away. /// /// Deliberately narrow. It covers exactly one accusation: "you did not have a chunk you /// were supposed to be holding". It does **not** cover the commitment-bound subtree audit, @@ -721,6 +736,19 @@ pub const AUDIT_FAILURE_TRUST_WEIGHT: f64 = 5.0; pub const RELEASE_SUSPEND_CLOSE_GROUP_STORAGE_PENALTY: bool = true; /// Environment override for [`RELEASE_SUSPEND_CLOSE_GROUP_STORAGE_PENALTY`], for a canary. +/// +/// Kept rather than removed with the rest of the bridge. This release does not flip the +/// constant, but the release after it does, and that is the moment the lever is most likely to +/// be needed. It suspends only the penalties this node hands out, so an emergency suspension +/// has to go to the fleet, not to the node being penalised. +/// +/// The one-way guard the previous release added is kept, and in this release it is ACTIVE: it +/// refuses to un-suspend while the release constant says to hold the penalty off, and this +/// release says exactly that. So the override can suspend and cannot un-suspend, and a host +/// setting it to `0` is told so and ignored — which is the point, because one host penalising +/// its close group for behaving as the release asked would slash all of them. It becomes inert +/// in the release that restores the penalty, and it stays there rather than being deleted so +/// that the meaning does not silently change if the constant ever goes back. pub const SUSPEND_CLOSE_GROUP_STORAGE_PENALTY_ENV: &str = "ANT_SUSPEND_UNHELD_CHUNK_PENALTY"; /// The live switch. @@ -743,7 +771,7 @@ pub fn apply_close_group_storage_penalty_policy() { apply_and_announce(RELEASE_SUSPEND_CLOSE_GROUP_STORAGE_PENALTY); return; }; - let suspended = match raw.trim().to_ascii_lowercase().as_str() { + let asked = match raw.trim().to_ascii_lowercase().as_str() { "1" | "true" | "yes" | "on" => true, "0" | "false" | "no" | "off" => false, other => { @@ -754,7 +782,22 @@ pub fn apply_close_group_storage_penalty_policy() { RELEASE_SUSPEND_CLOSE_GROUP_STORAGE_PENALTY } }; - apply_and_announce(suspended); + // Suspending is always allowed. Un-suspending is not, while the release says to hold + // off: this node would hand out full-weight trust failures to peers that are doing what + // the release asked of them, and they have no way to stop it or to see where it came + // from. + if !asked && RELEASE_SUSPEND_CLOSE_GROUP_STORAGE_PENALTY { + warn!( + "{SUSPEND_CLOSE_GROUP_STORAGE_PENALTY_ENV} asks this node to penalise peers for \ + not holding a close-group chunk, and this release holds that penalty off while \ + the fleet moves off the old chunk store. Ignoring it: a node giving chunks up \ + cannot stop a peer punishing it for doing so, so one host set this way would \ + slash its whole close group for behaving correctly." + ); + apply_and_announce(RELEASE_SUSPEND_CLOSE_GROUP_STORAGE_PENALTY); + return; + } + apply_and_announce(asked); } /// Set the switch and say so, once, where an operator will see it. @@ -793,7 +836,11 @@ fn apply_and_announce(suspended: bool) { } } -/// Set whether failing to hold a close-group chunk penalises. +/// Set whether the close-group unheld-chunk penalty is SUSPENDED. +/// +/// `true` holds the penalty off; `false` lets it be handed out. The parameter is the +/// suspension, not the punishment, and reading it the other way round is how a caller meaning +/// to switch punishing on switches it off. /// /// Startup applies the release policy through this. Tests that mean to exercise the /// penalty itself set it explicitly, so what they assert is not an accident of whichever @@ -802,7 +849,10 @@ pub fn set_close_group_storage_penalty_suspended(suspended: bool) { CLOSE_GROUP_STORAGE_PENALTY_SUSPENDED.store(suspended, Ordering::Relaxed); } -/// Whether failing to hold a close-group chunk currently penalises. +/// Whether the close-group unheld-chunk penalty is currently SUSPENDED. +/// +/// `true` means this node hands out no such penalty. The name says `suspended` and so does the +/// value; it is not a report of whether the lane penalises. #[must_use] pub fn close_group_storage_penalty_suspended() -> bool { CLOSE_GROUP_STORAGE_PENALTY_SUSPENDED.load(Ordering::Relaxed) @@ -1431,6 +1481,41 @@ mod tests { use super::*; use serial_test::serial; + /// The override can hold the penalty off. It cannot switch it back on. + /// + /// The direction is the whole point. A node giving chunks up during the migration cannot + /// stop a peer punishing it for doing so, so one host with this set the wrong way is not + /// a local choice about that host: it hands full-weight trust failures to every node in + /// its close group that is doing what the release asked. Set it right and it is a no-op + /// this release; set it wrong and it is a slashing incident nobody can trace. + #[test] + #[serial] + fn the_penalty_override_can_only_hold_the_penalty_off() { + // The switch is process-wide by design and this test is serialised, so nothing + // else is reading the variable or the switch while it runs. + std::env::set_var(SUSPEND_CLOSE_GROUP_STORAGE_PENALTY_ENV, "0"); + apply_close_group_storage_penalty_policy(); + assert_eq!( + close_group_storage_penalty_suspended(), + RELEASE_SUSPEND_CLOSE_GROUP_STORAGE_PENALTY, + "an operator switched the penalty back on and would slash their own close group" + ); + + std::env::set_var(SUSPEND_CLOSE_GROUP_STORAGE_PENALTY_ENV, "1"); + apply_close_group_storage_penalty_policy(); + assert!( + close_group_storage_penalty_suspended(), + "holding the penalty off is the safe direction and must still work" + ); + + std::env::remove_var(SUSPEND_CLOSE_GROUP_STORAGE_PENALTY_ENV); + apply_close_group_storage_penalty_policy(); + assert_eq!( + close_group_storage_penalty_suspended(), + RELEASE_SUSPEND_CLOSE_GROUP_STORAGE_PENALTY + ); + } + #[test] fn defaults_pass_validation() { let config = ReplicationConfig::default(); @@ -1461,6 +1546,28 @@ mod tests { /// One test rather than several, because the switch is process-wide: separate tests /// would race each other under the default parallel runner. + #[test] + #[serial] + fn this_release_still_holds_the_unheld_chunk_penalty_off() { + // Named on purpose, because the value matters and nothing else asserts it. This is + // the release that deletes the old chunk store, and the upgrade monitor picks the + // newest eligible release rather than the next one, so a node that was away while + // the migration ran arrives here having never migrated. It keeps its legacy store, + // which this build cannot read, so it cannot serve those chunks. Restoring the + // accusation here would slash that node in the release that stranded it. + // + // Restoring it is a one-line change in the release after this one, once the fleet + // has been observed clean for long enough to include the nodes that were away. + // Asked of the live switch after applying the policy rather than of the constant: + // clippy rejects an assertion on a constant, and what a node actually does is the + // better question. + apply_and_announce(RELEASE_SUSPEND_CLOSE_GROUP_STORAGE_PENALTY); + assert!( + close_group_storage_penalty_suspended(), + "this release must not restore the penalty; it can strand a node it then slashes" + ); + } + #[test] #[serial] fn the_unheld_chunk_penalty_switch_follows_the_release_it_is_compiled_into() { diff --git a/src/replication/mod.rs b/src/replication/mod.rs index 73d91f45..a202da17 100644 --- a/src/replication/mod.rs +++ b/src/replication/mod.rs @@ -994,7 +994,7 @@ const INBOUND_REPLICATION_SERIAL_QUEUE_CAPACITY: usize = 64; /// Maximum fresh-replication offers processed concurrently, away from the /// serial non-audit loop. /// -/// Fresh offers can perform an on-chain payment verification and a 4 MiB LMDB +/// Fresh offers can perform an on-chain payment verification and a 4 MiB /// write. Four workers keep that latency off the responder dispatch path while /// keeping concurrent EVM/storage pressure small and predictable. const FRESH_OFFER_WORKER_LIMIT: usize = 4; @@ -1128,7 +1128,7 @@ const FETCH_RESPONDER_MAX_OUTSTANDING_PER_PEER: u32 = 2; /// Maximum verification batches served concurrently. /// -/// LMDB point lookups are fast, but a batch can contain 8,192 of them. Two +/// Point lookups are fast, but a batch can contain 8,192 of them. Two /// workers isolate that synchronous work from message dispatch without turning /// large batches into an I/O fan-out throughput contest. const VERIFICATION_RESPONDER_WORKER_LIMIT: usize = 2; @@ -1486,13 +1486,13 @@ const BOOTSTRAP_DRAIN_CHECK_SECS: u64 = 5; /// observe the cancellation token and terminate before aborting it. /// /// Detached tasks are drained without a timeout because storage-capable work -/// may be awaiting a `spawn_blocking` LMDB operation, which continues running +/// may be awaiting a `spawn_blocking` storage operation, which continues running /// if its async waiter is dropped. const SHUTDOWN_TASK_DRAIN_TIMEOUT: Duration = Duration::from_secs(10); /// How often the responder rebuilds + rotates its storage commitment. /// -/// Each rebuild scans LMDB to compute leaf hashes; for ~10k keys this is +/// Each rebuild scans the store to compute leaf hashes; for ~10k keys this is /// sub-100ms (BLAKE3 + tree build). Retention is gossip-anchored, NOT /// rotation-anchored: the responder stays answerable for the current /// commitment plus every root it recently gossiped that is still in-window @@ -1702,7 +1702,7 @@ pub struct ReplicationEngine { identity: Arc, /// Responder-side commitment state (two-slot atomic rotation). /// - /// Periodically rebuilt from the live LMDB key set; gossiped on + /// Periodically rebuilt from the live key set; gossiped on /// outbound `NeighborSyncRequest`/`Response`; consulted by the /// commitment-bound audit handler. commitment_state: Arc, @@ -2328,14 +2328,14 @@ impl ReplicationEngine { /// /// This must be awaited before dropping the engine when the caller needs /// the `Arc` references held by background tasks to be - /// released (e.g. before reopening the same LMDB environment). + /// released (e.g. before reopening the same store). /// /// When this returns, no engine-spawned task still holds - /// `Arc` or `Arc`, and no LMDB blocking operation - /// (read or write, on either the chunk store or the paid-list + /// `Arc` or `Arc`, and no blocking storage operation + /// (read or write, against either the chunk store or the paid-list LMDB /// environment) is still running. Engine tasks race their work against /// the shutdown token; a dropped future may leave a `spawn_blocking` - /// LMDB transaction running detached, so this method additionally waits + /// operation running detached, so this method additionally waits /// for both storage layers to go quiescent before returning. pub async fn shutdown(&mut self) { self.shutdown.cancel(); @@ -2376,11 +2376,12 @@ impl ReplicationEngine { // All producers have stopped, so close and drain their detached work. // A started storage operation must run to completion: dropping an async // waiter does not cancel `spawn_blocking`, and would let shutdown return - // while an LMDB transaction still owns the environment. + // while a blocking storage operation is still running. // - // Deliberately unbounded: the LMDB contract requires every worker to - // release its `Arc` before the caller may reopen the - // environment, and a timeout here could return with one still held. + // Deliberately unbounded: every worker has to release its + // `Arc` before the caller may reopen the store, whose lock + // admits one process at a time, and a timeout here could return with one + // still held. // What makes that safe is that every detached task is now guaranteed to // finish — the pools above are closed, stale work is shed at dequeue, // and the one genuinely unbounded await (payment verification) races @@ -2389,7 +2390,7 @@ impl ReplicationEngine { self.detached_task_tracker.wait().await; // Every producer is gone, but a select! racing the shutdown token may - // have dropped a future while it awaited an LMDB `spawn_blocking` op + // have dropped a future while it awaited a storage `spawn_blocking` op // (fetch `storage.put`, prune `storage.delete` / // `paid_list.remove_batch`, verification `paid_list.insert`). The // detached blocking closure owns a cloned `Env`; wait for both @@ -2526,12 +2527,12 @@ impl ReplicationEngine { // so those waiters would drain only at the probe timeout // (roughly `queued / per-target-limit` probes deep) while // `detached_task_tracker.wait()` — deliberately unbounded - // for the LMDB contract — held shutdown open. + // for the storage contract — held shutdown open. // // Dropping this future mid-probe is safe and is the same // shape the neighbor-sync round uses: a parked coordinator // acquire releases its counted reference via - // `ReferenceGuard`, and a dropped LMDB `spawn_blocking` is + // `ReferenceGuard`, and a dropped storage `spawn_blocking` is // covered by the storage-quiescence wait in `shutdown`. tokio::select! { () = shutdown.cancelled() => {} @@ -3388,7 +3389,7 @@ impl ReplicationEngine { /// /// Phase 3 of the v12 storage-bound audit. Once per /// [`COMMITMENT_ROTATION_INTERVAL_SECS`], the responder reads the - /// current LMDB key set, builds a Merkle tree (for content-addressed + /// current key set, builds a Merkle tree (for content-addressed /// chunks `bytes_hash == key`, so no chunk re-read is needed), signs /// the root with the node's `MlDsaSecretKey`, and rotates the result /// into `commitment_state`. Old `previous` slot is dropped by the @@ -4374,7 +4375,7 @@ struct ReplicationMessageHandlerContext { /// The engine's shutdown token, for detached responder work. /// /// Workers on [`Self::detached_task_tracker`] race this around their - /// *network* phase only — never around an LMDB `spawn_blocking` await, + /// *network* phase only — never around a storage `spawn_blocking` await, /// where dropping the awaiter would detach a live transaction. This is /// what lets `shutdown()` keep its unbounded `tracker.wait()` and still /// terminate: the wait stays safe because it is now guaranteed finite. @@ -5600,7 +5601,7 @@ async fn handle_replication_message( /// is guaranteed to end. /// /// Deliberately NOT applied to `storage.put`: that awaits `spawn_blocking`, so -/// dropping its awaiter would detach a live LMDB transaction and break the +/// dropping its awaiter would detach a live storage operation and break the /// very contract the unbounded wait exists to uphold. async fn verify_payment_until_shutdown( payment_verifier: &Arc, @@ -5830,7 +5831,7 @@ async fn refuse_stranded_fresh_offers( /// /// This runs on the serial non-audit message loop, so it must stay cheap: every /// path here is a set insert, a permit try, or a small response send. The offer -/// itself — an on-chain payment verification and a multi-MiB LMDB write — always +/// itself — an on-chain payment verification and a multi-MiB write — always /// runs on a tracked worker task, never inline, because stalling this loop backs /// up the inbound queue and ultimately drops replication messages wholesale. /// @@ -5987,7 +5988,7 @@ async fn dispatch_fresh_offer( /// /// Split out so `dispatch_fresh_offer` stays a readable admission decision. /// A started handler is never cancelled: `storage.put()` awaits -/// `spawn_blocking`, and dropping that awaiter would detach the live LMDB +/// `spawn_blocking`, and dropping that awaiter would detach the live storage /// transaction. Shutdown responsiveness comes from the closed worker semaphore /// and from `handle_fresh_offer` racing the token around payment verification. /// @@ -7116,9 +7117,12 @@ fn request_is_stale(received_at: Instant, timeout: Duration) -> bool { enum FetchFault { /// The peer does not hold a chunk it was expected to hold. /// - /// This is the lane the release withholds, because a node part-way through moving - /// off the legacy store answers exactly this way about chunks it has legitimately - /// given up. + /// This is the lane the migration releases withhold, because a node part-way through + /// moving off the old store answers exactly this way about chunks it had legitimately + /// given up. It is STILL withheld here: this release deletes the old store, and a node + /// that was away while the migration ran arrives holding one it cannot read, so + /// accusing it in the release that stranded it would slash it for a state it had no + /// chance to leave. The release after this one restores it. UnheldChunk, /// The peer's own storage failed, or served bytes that no longer hash to their /// address. @@ -9098,7 +9102,7 @@ async fn execute_single_fetch( if let Err(e) = storage.put(&resp_key, &data).await { // The bytes arrived and passed the content-address // check, so the source did its job; the failure is - // entirely local (disk-full, or an LMDB error). Any + // entirely local (disk-full, or a storage error). Any // valid source must serve identical content, so trying // the next one cannot cure a local error — it only // re-downloads the same chunk into the same store. @@ -10100,7 +10104,7 @@ async fn write_retention_atomic(path: &Path, bytes: Vec) -> bool { } } -/// Read the current LMDB key set, build + sign a fresh +/// Read the current key set, build + sign a fresh /// `StorageCommitment`, and rotate it into `state` as the new `current`. /// The prior `current` is demoted to `previous`; the prior `previous` is /// dropped (per `ResponderCommitmentState::rotate`). @@ -10124,13 +10128,14 @@ async fn rebuild_and_rotate_commitment( p2p: &Arc, config: &Arc, ) -> Result<()> { - // Not `all_keys()`. While the node is bridging off the legacy store these are the - // same thing, but once it has settled on what it can hold this narrows to the - // file-backed set, which is what stops it claiming keys it is about to give up. It is - // also what lets `is_held` eventually go false for those keys, which is the gate on - // removing the legacy environment at all. + // `all_keys()` is what the store can answer for: it already drops a file marked suspect or + // known-wrong, so a name that cannot be read is not offered here. It is NOT the commitment + // set — that is narrowed to the keys this node is still responsible for, by the filter a + // few lines below, and not by this call. The comment that used to be here said "not + // `all_keys()`" immediately above the call to it, which is the sort of thing that sends + // somebody to change the wrong layer. let stored_keys = storage - .committable_keys() + .all_keys() .await .map_err(|e| Error::Storage(format!("commitment build: read keys: {e}")))?; @@ -10200,7 +10205,6 @@ async fn rebuild_and_rotate_commitment( (stays answerable until its gossip TTL lapses, bytes still on disk)" ); state.retire_current(); - storage.note_commitment_rebuilt(); return Ok(()); } @@ -10287,9 +10291,6 @@ async fn rebuild_and_rotate_commitment( // committed key set is frozen here for many rotations. Without this, // the no-op guard would pin a stale slot — and its key — forever. state.age_out(); - // The advertised commitment already equals the committable set, which is - // exactly what the retirement gate is counting. - storage.note_commitment_rebuilt(); return Ok(()); } } @@ -10313,10 +10314,6 @@ async fn rebuild_and_rotate_commitment( let pointer_count = built.tree().pointer_count(); state.rotate(built); info!("Storage commitment rotated: hash={hash} key_count={key_count} pointers={pointer_count}"); - // Counted only on the paths where the advertised commitment now genuinely reflects - // the committable set, never merely on having read it. The retirement gate is what - // consumes this, and it authorises deleting the legacy store. - storage.note_commitment_rebuilt(); Ok(()) } @@ -10357,15 +10354,14 @@ mod tests { /// to a response is what the classification above rests on. /// /// A key the peer does not hold reads as `Ok(None)`. A read that fails, whether from - /// an I/O fault or a failed integrity check, reads as `Err`. Nothing in the migration - /// turns the first into the second. + /// an I/O fault or a failed integrity check, reads as `Err`. Nothing turns the first + /// into the second. #[tokio::test] async fn a_missing_key_reads_as_a_plain_miss_and_a_failed_read_as_a_fault() { let dir = tempfile::tempdir().expect("temp dir"); - let storage = crate::storage::LmdbStorage::new(crate::storage::LmdbStorageConfig { + let storage = crate::storage::ChunkStore::new(crate::storage::ChunkStoreConfig { root_dir: dir.path().to_path_buf(), verify_on_read: true, - max_map_size: 0, disk_reserve: 0, }) .await diff --git a/src/replication/pruning.rs b/src/replication/pruning.rs index 9d213126..aa503757 100644 --- a/src/replication/pruning.rs +++ b/src/replication/pruning.rs @@ -1241,53 +1241,6 @@ async fn collect_record_prune_proofs( present_by_key } -/// Prove that other nodes actually hold `keys`, by cryptographic challenge. -/// -/// Exposed for the storage migration, which has to answer the same question the pruner -/// answers before it deletes: is this chunk somewhere else? It deliberately reuses this -/// path rather than the cheaper `VerificationRequest`, because that one carries a -/// self-reported `present: bool` and a node that has silently lost a chunk will still say -/// yes. Here the peer has to return `compute_audit_digest(nonce, peer, key, bytes)` over a -/// nonce it has never seen, which it cannot do without the bytes. -/// -/// Returns, per key, the set of peers that proved possession. The caller decides how many -/// are enough; [`prune_proofs_needed`] is the rule the pruner uses. -pub(crate) async fn prove_peers_hold_records( - keys_by_peer: &HashMap>, - local_stored_key_count: usize, - storage: &Arc, - p2p_node: &Arc, - config: &ReplicationConfig, - sync_state: &Arc>, - audit_challenge_coordinator: &Arc, -) -> HashMap> { - if keys_by_peer.is_empty() { - return HashMap::new(); - } - let candidates: Vec = { - let mut by_key: HashMap> = HashMap::new(); - for (peer, keys) in keys_by_peer { - for key in keys { - by_key.entry(*key).or_default().push(*peer); - } - } - by_key - .into_iter() - .map(|(key, target_peers)| RecordPruneCandidate { key, target_peers }) - .collect() - }; - collect_record_prune_proofs( - &candidates, - local_stored_key_count, - storage, - p2p_node, - config, - sync_state, - audit_challenge_coordinator, - ) - .await -} - async fn revalidated_fast_prune_keys( candidates: &[FastPruneCandidate], ctx: &PrunePassContext<'_>, diff --git a/src/replication/storage_commitment_audit.rs b/src/replication/storage_commitment_audit.rs index c20b8e60..ec70ca81 100644 --- a/src/replication/storage_commitment_audit.rs +++ b/src/replication/storage_commitment_audit.rs @@ -1349,7 +1349,7 @@ pub async fn handle_subtree_challenge( pub struct Round1Work { /// What to send back. pub response: SubtreeAuditResponse, - /// Chunk content read from LMDB and hashed BEFORE this response was + /// Chunk content read from the store and hashed BEFORE this response was /// produced. /// /// Counted on the rejecting paths too, which is the point. A subtree is read @@ -1540,7 +1540,7 @@ async fn subtree_challenge_response( continue; } // Charge the fixed cost of ATTEMPTING a leaf before the read, because - // it is owed whether or not the read succeeds: the LMDB lookup and its + // it is owed whether or not the read succeeds: the lookup and its // retries, and the blocking-task round trip below. Charging only // content bytes left both a failing leaf and a tiny one nearly free, // and nothing bounds a chunk from below, so a commitment of a million @@ -1802,7 +1802,7 @@ pub async fn handle_subtree_slice_challenge_with_pointers( }; // Coalesce openings by key, preserving first-seen order and deduplicating - // block indices per key, so each committed chunk is read from LMDB and hashed + // block indices per key, so each committed chunk is read from the store and hashed // at most once even when the auditor opens several of its blocks (the normal // random + final pair, or a forged duplicate). Without this a ten-opening // request could re-read and re-hash the same chunk ten times. @@ -1984,7 +1984,7 @@ async fn serve_committed_key_openings( } // Persistent transient read error after retries → do NOT brand the peer a // deleter. Reject `Transient`; the auditor routes it to the timeout lane - // so a flaky LMDB read never manufactures a confirmed possession failure + // so a flaky read never manufactures a confirmed possession failure // on an honest holder (which also gains no credit). Err(e) => { warn!( diff --git a/src/storage/chunk_store.rs b/src/storage/chunk_store.rs index fe297705..3fa16ef1 100644 --- a/src/storage/chunk_store.rs +++ b/src/storage/chunk_store.rs @@ -1,111 +1,203 @@ -//! The node's chunk store: a file store, plus the legacy LMDB environment for as long -//! as one still exists on disk. +//! One immutable file per chunk, content-addressed, with the filesystem as the +//! only authority. //! -//! Every caller in the node talks to this type and sees **one** key set. That is the -//! detail that keeps quoting, commitments, hints, audits and pruning coherent while a -//! chunk moves from LMDB to a file: the backing changes, the logical key set does not. +//! ```text +//! {root}/chunks/ store root +//! {root}/chunks/layout.json versioned layout marker +//! {root}/chunks/.lock advisory single-process guard +//! {root}/chunks//<64-hex> xy = the LAST two hex characters of the address +//! {root}/chunks//.tmp.. an in-flight write, in the destination directory +//! ``` //! -//! There is one deliberate asymmetry, and it is the whole safety argument of the -//! migration. Serving reads the **union**, so the node answers for everything it ever -//! committed to. The commitment builder reads only the **file-backed** set once the node -//! has settled on what it will keep, so the node stops claiming keys it is about to give -//! up. Between those two, a node is at worst over-honest: it serves more than it claims. +//! # Why the *last* two hex characters +//! +//! A node holds keys for which it is among the [`CLOSE_GROUP_SIZE`] closest, so its +//! holdings share roughly `log2(N / CLOSE_GROUP_SIZE)` leading bits with its own node +//! ID, and that shared prefix grows as the network grows. Sharding on a prefix therefore +//! does not degrade, it collapses: at ~800 nodes a two-hex prefix already resolves to +//! about two distinct directories, and past a million nodes even a four-hex prefix +//! resolves to one. Close-group membership constrains the leading bits and places no +//! constraint at all on the trailing ones, and the address is a BLAKE3 output, so the +//! last byte is uniform by construction at every network size. +//! +//! 256 shards keeps a 24 GiB node at ~23 files per directory and a 1 TiB node at ~977, +//! for 1 MiB of directory inodes. The scheme and depth are recorded in `layout.json` at +//! creation so a future layout can be detected rather than silently misread. +//! +//! # Why lowercase hex names +//! +//! NTFS and default APFS fold case. Under an encoding with both cases (base64url, +//! base58) two distinct 32-byte keys can share one case-folded filename, which is a +//! silent overwrite. Hex has one case-folded form per key, and no hex string can ever +//! spell a reserved Windows device name (`CON`, `NUL`, `AUX`, `COM1`, ...) because none +//! of those letters is in `0-9a-f`. The full 64-character key stays in the filename, so +//! a `find` over the tree recovers the whole store even if the directory layer is lost. +//! +//! [`CLOSE_GROUP_SIZE`]: crate::ant_protocol::CLOSE_GROUP_SIZE -use crate::ant_protocol::XorName; +use crate::ant_protocol::{XorName, MAX_CHUNK_SIZE, XORNAME_LEN}; use crate::error::{Error, Result}; -use crate::logging::{debug, error, info, warn}; -use crate::storage::file_store::{FileStore, FileStoreConfig}; -use crate::storage::lmdb::{LmdbStorage, LmdbStorageConfig}; -use crate::storage::migration::{ - CopyReport, MigrationConfig, MigrationPhase, MigrationState, REQUIRED_REBUILDS_BEFORE_RETIRE, -}; +use crate::logging::{debug, info, trace, warn}; use crate::storage::StorageStats; -use std::collections::{BTreeMap, BTreeSet}; -use std::io::Write; +use fs2::FileExt; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeSet, HashMap, HashSet}; +use std::fs::{File, OpenOptions}; +use std::io::{ErrorKind, Read, Write}; use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; -use std::time::Duration; -use tokio_util::sync::CancellationToken; +use std::time::{Duration, Instant}; +use tokio::task::spawn_blocking; +use tokio_util::task::TaskTracker; -/// Directory name of the legacy LMDB environment, under the node root. -pub const LEGACY_ENV_DIR: &str = "chunks.mdb"; +/// Directory under the node root that holds the chunk files. +pub const CHUNKS_DIR_NAME: &str = "chunks"; -/// Suffix for a legacy environment that has been retired but not yet deleted. -pub const RETIRED_SUFFIX: &str = ".retired"; +/// Name of the layout marker written once at store creation. +pub const LAYOUT_FILE_NAME: &str = "layout.json"; -/// Written inside a chunk environment directory once it has been retired. -/// -/// The rename that moves the environment aside cannot be shown to be durable off Unix: -/// there is no way to flush a directory through the standard library, and `MoveFileEx` is -/// not documented as durable at return without a flag std does not use. So a power loss -/// can bring the directory back under its old name with its contents already deleted, and -/// a node that tried to open that would fail to start. -/// -/// This file is what makes that unambiguous, and it is *inside* the directory rather than -/// beside it so that it travels with it: a directory that reverts to its old name reverts -/// carrying its own evidence. It is created with the same create-and-flush that publishes -/// a chunk, which is documented as durable everywhere, and only after the rename has -/// already succeeded. So a directory holding it has been retired, whatever it is called, -/// and one that does not is a live environment and is opened normally. -/// -/// Deliberately not a file beside the environment. A marker that can outlive the thing it -/// describes has to be cancelled, cancellation can fail or be lost, and a stale one would -/// authorise deleting an environment that had since taken a chunk. -const RETIRED_MARKER: &str = "RETIRED"; +/// Name of the advisory single-process lock file. +const LOCK_FILE_NAME: &str = ".lock"; -/// How many times the background reaper retries deleting a retired directory. +/// Prefix that marks an in-flight write. Never a valid chunk name (chunk names are +/// exactly [`CHUNK_NAME_LEN`] lowercase hex characters, and `.` is not hex). +const TEMP_PREFIX: &str = ".tmp."; + +/// Number of shard directories. One level, `00` through `ff`. +const SHARD_COUNT: usize = 256; + +/// Length of a chunk filename: the full address in lowercase hex. +const CHUNK_NAME_LEN: usize = XORNAME_LEN * 2; + +/// How often to re-query available disk space, in seconds. /// -/// Generous, because giving up strands the disk until the next restart and the thread -/// costs nothing while it sleeps. With the backoff below this keeps trying for about a -/// day. -const RETIRED_DELETE_ATTEMPTS: u32 = 60; +/// Matches the LMDB store's cadence so the capacity predicate behaves identically +/// for callers that only ask "is there room at all". +const DISK_CHECK_INTERVAL_SECS: u64 = 5; -/// Base wait between those attempts, multiplied by the attempt number up to the cap. -const RETIRED_DELETE_BACKOFF: Duration = Duration::from_secs(10); +/// Allocation granularity assumed when charging a pending write against free space. +/// +/// Every filesystem we support allocates in units of at least 4 KiB, so a write of +/// `n` bytes consumes at least `ceil(n / 4096) * 4096`. One extra unit covers the +/// directory entry and inode. +const ALLOC_UNIT: u64 = 4096; -/// The longest the reaper waits between attempts. -const RETIRED_DELETE_BACKOFF_MAX: Duration = Duration::from_mins(30); +/// How many times a publish retries a transient Windows sharing violation. +const RENAME_RETRY_ATTEMPTS: u32 = 5; -/// How many retired directories may be waiting to be deleted before the node stops -/// finding new names for them. Far more than a node should ever accumulate. -const MAX_TOMBSTONES: u32 = 64; +/// Base backoff between those retries; the wait grows linearly with the attempt. +const RENAME_RETRY_BACKOFF: Duration = Duration::from_millis(20); -/// The legacy environment's data file. Its presence is what says a node still has one. -const LEGACY_DATA_FILE: &str = "data.mdb"; +/// Longest absolute path a chunk file may need, checked once at open. +/// +/// Windows caps a non-verbatim path at `MAX_PATH` (260) including the terminating NUL. +/// Rust's standard library transparently switches to the `\\?\` verbatim form for long +/// absolute paths, so this is a warning rather than a hard failure, but an operator who +/// buries the node root ten directories deep should hear about it before the first write +/// fails rather than after. +#[cfg(windows)] +const WINDOWS_PATH_WARN_LEN: usize = 240; + +/// The on-disk layout marker. +/// +/// Written once when the store directory is created and read on every subsequent open. +/// Nothing in this survey of comparable stores (IPFS flatfs, Storj, borgbackup) shipped +/// an in-place re-sharder, and all three paid for it. Recording the scheme costs one +/// small file and is the difference between changing the default later and never being +/// able to. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct StoreLayout { + /// Marker schema version. A store written by a newer schema is refused. + pub schema: u32, + /// How a chunk address maps to a shard directory. + pub scheme: String, + /// How many hex characters of the address name the shard directory. + pub shard_chars: u8, + /// How many directory levels of sharding. + pub depth: u8, + /// How a chunk address maps to a filename. + pub name_encoding: String, +} -/// How many times retirement retries taking sole ownership of the legacy handle before -/// giving up for this tick. -const RETIRE_UNWRAP_ATTEMPTS: u32 = 20; +/// Marker schema this build writes and understands. +const LAYOUT_SCHEMA: u32 = 1; +/// Shard scheme this build implements: the trailing hex characters of the address. +const LAYOUT_SCHEME_SUFFIX_HEX: &str = "suffix-hex"; +/// Filename encoding this build implements. +const LAYOUT_NAME_LOWER_HEX: &str = "lower-hex"; -/// How long to wait between those attempts. -const RETIRE_UNWRAP_BACKOFF: Duration = Duration::from_millis(100); +impl Default for StoreLayout { + fn default() -> Self { + Self { + schema: LAYOUT_SCHEMA, + scheme: LAYOUT_SCHEME_SUFFIX_HEX.to_string(), + shard_chars: 2, + depth: 1, + name_encoding: LAYOUT_NAME_LOWER_HEX.to_string(), + } + } +} -/// How many chunks the verification pass checks between progress lines. -const VERIFY_LOG_EVERY: u64 = 2000; +impl StoreLayout { + /// Return an error unless this build can read a store written with this layout. + fn check_supported(&self) -> Result<()> { + if self.schema > LAYOUT_SCHEMA { + return Err(Error::Storage(format!( + "Chunk store layout schema {} is newer than this build understands ({LAYOUT_SCHEMA}). \ + Refusing to open rather than misread the store.", + self.schema + ))); + } + if self.scheme != LAYOUT_SCHEME_SUFFIX_HEX { + return Err(Error::Storage(format!( + "Chunk store uses shard scheme '{}', this build implements '{LAYOUT_SCHEME_SUFFIX_HEX}'", + self.scheme + ))); + } + if self.shard_chars != 2 || self.depth != 1 { + return Err(Error::Storage(format!( + "Chunk store uses {} shard characters at depth {}, this build implements 2 at depth 1", + self.shard_chars, self.depth + ))); + } + if self.name_encoding != LAYOUT_NAME_LOWER_HEX { + return Err(Error::Storage(format!( + "Chunk store names files with '{}', this build implements '{LAYOUT_NAME_LOWER_HEX}'", + self.name_encoding + ))); + } + Ok(()) + } +} -/// How many per-key critical sections the facade keeps. +/// What the store can say about free space right now. +/// +/// Three answers, because deciding how long to stand down needs the distinction: a full +/// disk is a standing condition worth waiting minutes on, while a failed query may have +/// cleared by the next attempt and must not be treated as one. /// -/// Keyed on the address's LAST byte, for the same reason the shard directories are: a -/// node's keys share their leading bytes, so lanes keyed on the first byte would all -/// collapse into one. -const KEY_LOCK_LANES: usize = 256; +/// Lived alongside the LMDB store until that was removed. It was never about LMDB. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CapacityVerdict { + /// Available space is at or above the configured reserve. That is what the query + /// establishes, and possibly from the TTL cache, not a promise the next write succeeds. + Writable, + /// Available space is below the configured reserve. + Full, + /// The query itself failed, so nothing is known about available space. + Unknown, +} /// Configuration for [`ChunkStore`]. #[derive(Debug, Clone)] pub struct ChunkStoreConfig { - /// Node root directory. + /// Node root directory. The store lives at `{root_dir}/chunks/`. pub root_dir: PathBuf, /// Verify `BLAKE3(content) == address` on read. pub verify_on_read: bool, - /// Explicit LMDB map size cap in bytes, used only while a legacy environment exists. - /// - /// Dies with LMDB. Kept so an operator's existing `storage.db_size_gb` still means - /// what it meant during the bridge. - pub max_map_size: usize, - /// Minimum free disk space to preserve on the storage partition. + /// Free bytes to keep on the storage partition. Writes are refused below this. pub disk_reserve: u64, - /// Migration controls. - pub migration: MigrationConfig, } impl Default for ChunkStoreConfig { @@ -113,16 +205,13 @@ impl Default for ChunkStoreConfig { Self { root_dir: PathBuf::from(".ant/chunks"), verify_on_read: true, - max_map_size: 0, disk_reserve: crate::storage::DEFAULT_DISK_RESERVE, - migration: MigrationConfig::default(), } } } impl ChunkStoreConfig { - /// A test-friendly default with the disk reserve disabled, so unit tests do not - /// depend on the host having spare gigabytes. + /// The shipped defaults with the disk reserve removed, for tests on small volumes. #[cfg(any(test, feature = "test-utils"))] #[must_use] pub fn test_default() -> Self { @@ -133,706 +222,890 @@ impl ChunkStoreConfig { } } -/// The legacy environment and the keys only it still holds. -#[derive(Clone)] -struct Legacy { - /// The LMDB handle. - lmdb: Arc, - /// Keys in the legacy environment that are **not** in the file store. - /// - /// Kept in memory so the union view costs nothing on the hot paths: `exists` and - /// `current_chunks` never touch LMDB, and `all_keys` merges two already-sorted - /// sequences. It is derived at open (LMDB keys minus file keys) and maintained by - /// every write, copy and delete. - only: Arc>>, - /// Writes that have started and whose outcome is not yet known. - /// - /// A write into the legacy environment runs on a blocking thread that outlives the - /// future waiting for it, so a shutdown can leave the environment holding a chunk - /// while nothing ran to record it. A key in neither view is what retirement destroys, - /// so every write announces itself here first. - /// - /// Deliberately NOT part of what the node says it holds. This is a note to itself - /// that something is in flight, not a claim: `exists`, `all_keys`, the commitment, the - /// quote count and the pruner all ignore it. It vetoes retirement, and the driver - /// resolves each entry against what is actually on disk. - /// - /// How many writes were made without a rollback copy of the chunk in the environment. - /// - /// The rollback copy is best effort by design, and the ADR says so: a bridging node - /// whose environment has no reusable page keeps serving from files and simply has no - /// second copy to roll back to. What was missing was any way to ask how often that - /// happened. One `warn!` per chunk is not an answer to "how many nodes on this fleet - /// are actually keeping a rollback copy", which is the question the second release - /// turns on, and on a node with no free pages it is also a line per chunk forever. - /// - /// Writes, not distinct chunks: two attempts at one address count twice, and a later - /// attempt that succeeds does not count back down. Counted before the file half runs, - /// so a write that then fails outright is counted too. Keeping a set of addresses - /// instead would be exact and would also mean holding millions of them in memory to - /// answer a question that a rate answers. Read it as "this node is failing to keep - /// rollback copies, this often", not as a chunk count. - skipped_rollback_copies: Arc, - - /// Counted, not a set, for the reason the file store's `writing` map is counted. - /// Cancellation can release the facade's key lane while the blocking half survives, so - /// a second write for the same key can start behind the first. With one entry between - /// them, whichever returned first would clear it while the other was still queued, and - /// a delete arriving in that window would see no announcement, skip draining the - /// environment, and let the surviving write land afterwards and put the key back. - pending: Arc>>, +/// Outcome of a single write attempt, used to keep the duplicate accounting honest. +enum PutOutcome { + /// The chunk was newly published. + New, + /// The chunk was already on disk. + Duplicate, } -impl Legacy { - /// Note that a chunk went to files alone, and say whether to log it. - /// - /// Throttled by powers of ten. The condition is usually all-or-nothing, so the first - /// few lines say it started and the later ones say it is still going without becoming - /// the log. - fn note_skipped_rollback_copy(&self) -> Option { - let count = self - .skipped_rollback_copies - .fetch_add(1, std::sync::atomic::Ordering::Relaxed) - .saturating_add(1); - let round = matches!( - count, - 10 | 100 | 1_000 | 10_000 | 100_000 | 1_000_000 | 10_000_000 - ); - (count <= 3 || round).then_some(count) - } +/// Snapshot of free space, plus what has been written since it was taken. +#[derive(Debug)] +struct CapacitySnapshot { + /// When `available` was measured. `None` means never. + measured_at: Option, + /// Free bytes reported by the filesystem at `measured_at`. + available: u64, + /// Bytes published since `measured_at`, charged against `available`. + /// + /// Cleared by a fresh measurement, which already accounts for them. + written_since: u64, + /// Bytes reserved by writes that have not landed yet. + /// + /// Deliberately **not** cleared by a measurement: a `statvfs` taken while writes are + /// in flight reports space those writes are about to consume, so forgetting their + /// reservations at that moment would hand the same bytes out twice. That is precisely + /// the over-admission the reservation exists to prevent. + in_flight: u64, +} + +/// Size-aware free-space predicate with a short-lived cache. +/// +/// Free bytes alone stopped being a sufficient answer the moment chunks became files: +/// a caller wants to know whether *this* write fits, not whether the disk is non-empty. +/// The cache keeps the common case at one `statvfs` per interval while staying correct +/// under a burst, because bytes written since the measurement are charged against it. +#[derive(Debug)] +struct CapacityGuard { + /// Directory whose partition is measured. + dir: PathBuf, + /// Free bytes to keep unused. + reserve: u64, + /// The cached measurement. + snapshot: parking_lot::Mutex, +} - /// Announce a write into the environment, or note a second one for the same key. - fn announce(&self, address: &XorName) { - *self.pending.write().entry(*address).or_insert(0) += 1; +impl CapacitySnapshot { + /// Free bytes, less everything written or promised since the measurement. + fn free_estimate(&self) -> u64 { + self.available + .saturating_sub(self.written_since) + .saturating_sub(self.in_flight) } +} - /// Retire one announcement, leaving any other for the same key still standing. - fn announced_write_finished(&self, address: &XorName) { - let mut pending = self.pending.write(); - let Some(count) = pending.get_mut(address) else { - return; - }; - *count = count.saturating_sub(1); - if *count == 0 { - pending.remove(address); +impl CapacityGuard { + /// Create a guard over the partition hosting `dir`. + fn new(dir: PathBuf, reserve: u64) -> Self { + Self { + dir, + reserve, + snapshot: parking_lot::Mutex::new(CapacitySnapshot { + measured_at: None, + available: 0, + written_since: 0, + in_flight: 0, + }), } } -} -/// Content-addressed chunk storage. -pub struct ChunkStore { - /// The file store. Always present, always the write target. - files: Arc, - /// The legacy environment, until it is retired. - legacy: parking_lot::RwLock>, - /// Excludes retirement while any operation that touches the legacy environment runs. - /// - /// Reads, writes and deletes take it shared for their whole duration; retirement takes - /// it exclusively before it takes the environment away. Sole ownership of the handle - /// is not enough on its own: a read that has decided the file store cannot answer, and - /// has not yet taken a legacy handle, holds nothing and would be invisible to that - /// check. Nor would holding a handle throughout do instead, because on a busy node - /// there would always be one and retirement would never see the environment - /// unreferenced. Retirement also waits for the environment to go idle, which never - /// happens if new work can keep starting in it. - /// - /// A shared/exclusive lock states the actual requirement, and because it is fair, a - /// waiting retirement stops new work starting rather than starving behind it. - retirement: tokio::sync::RwLock<()>, - /// Where the legacy environment lives. - legacy_env_dir: PathBuf, - /// Store configuration. - config: ChunkStoreConfig, - /// The persisted migration marker. - state: parking_lot::RwLock, - /// One lock per shard, held across a whole logical key transition. - /// - /// The file store has its own lane locks, but those only make a single file write - /// atomic. The races that matter here span two stores and an await point: the copier - /// reads a chunk out of LMDB, the pruner deletes that chunk from both stores, and - /// then the copier's write lands and resurrects it. One critical section per key, - /// held across put, delete and copy, is what closes that. - key_locks: Vec>, -} + /// Bytes actually consumed on disk by a payload of `len` bytes. + fn charge(len: u64) -> u64 { + // Round the payload up to the allocation unit, then add one unit for the + // directory entry and inode. + len.div_ceil(ALLOC_UNIT) + .saturating_mul(ALLOC_UNIT) + .saturating_add(ALLOC_UNIT) + } -impl ChunkStore { - /// Open the store under `config.root_dir`. - /// - /// Opens the legacy environment only if one is already on disk. A fresh node never - /// creates one, so it never pays for a memory map it will not use. + /// Free bytes right now, or `None` if the question could not be answered. /// - /// # Errors + /// Deliberately separate from [`Self::measure`], which folds a failure into an error + /// the caller cannot tell from "below the reserve". + fn measure_available(&self) -> Option { + let mut snapshot = self.snapshot.lock(); + match self.measure(&mut snapshot) { + Ok(()) => Some(snapshot.free_estimate()), + Err(_) => None, + } + } + + /// Query the filesystem and refresh the snapshot. + fn measure(&self, snapshot: &mut CapacitySnapshot) -> Result<()> { + let available = fs2::available_space(&self.dir) + .map_err(|e| Error::Storage(format!("Failed to query available disk space: {e}")))?; + snapshot.available = available; + // Reservations survive: their bytes are not on the platter yet, so the fresh + // measurement does not include them. + snapshot.written_since = 0; + snapshot.measured_at = Some(Instant::now()); + Ok(()) + } + + /// Test `needed` against the snapshot, refreshing it if it is stale or short. /// - /// Returns [`Error::Storage`] if either store cannot be opened. - pub async fn new(config: ChunkStoreConfig) -> Result { - let files = Arc::new( - FileStore::new(FileStoreConfig { - root_dir: config.root_dir.clone(), - verify_on_read: config.verify_on_read, - disk_reserve: config.disk_reserve, - }) - .await?, - ); + /// Only *passing* results are cached, so a low-space condition is rechecked on every + /// call and freed space is noticed promptly. + fn admit(&self, snapshot: &mut CapacitySnapshot, needed: u64) -> Result<()> { + let want = self.reserve.saturating_add(Self::charge(needed)); - // Before anything looks at the legacy environment: a directory carrying its own - // retirement mark is the remains of a removal a power loss interrupted, and is - // moved aside rather than opened. - let openable = finish_interrupted_retirement(&config.root_dir); - let legacy_env_dir = config.root_dir.join(LEGACY_ENV_DIR); - let legacy = - if openable == LiveEnvironment::WhateverIsOnDisk && legacy_present(&config.root_dir)? { - Some(Self::open_legacy(&config, &files).await?) - } else { - None - }; + let cache_fresh = snapshot + .measured_at + .is_some_and(|t| t.elapsed().as_secs() < DISK_CHECK_INTERVAL_SECS); + if cache_fresh && snapshot.free_estimate() >= want { + return Ok(()); + } - let phase = if legacy.is_some() { - MigrationPhase::Bridging - } else { - MigrationPhase::FilesOnly - }; - let mut state = MigrationState::load_or_create(&config.root_dir, phase); - let legacy_only = Self::legacy_only_count(legacy.as_ref()); - - // The filesystem is the authority on whether a legacy environment exists; the - // marker only records decisions. Reconcile rather than trust. - if legacy.is_none() && state.phase != MigrationPhase::FilesOnly { - info!("No legacy chunk environment on disk; the migration is already complete"); - state.phase = MigrationPhase::FilesOnly; - if let Err(e) = state.save(&config.root_dir) { - warn!("Could not persist the migration marker: {e}"); - } - } else if legacy.is_some() - && state.phase == MigrationPhase::Committed - && files.current_chunks().unwrap_or(0) < state.kept_key_count - { - // The marker says this node already settled on what it would keep, but the - // file store holds less than it recorded keeping. Something outside the node - // changed the data directory, and trusting the marker here would skip the - // copier, the shed rules and their rank checks on the way to deleting the - // legacy environment. The filesystem wins. - warn!( - "The migration marker says this node kept {} chunk(s) but the file store \ - holds {}. Restarting the migration from the copying stage.", - state.kept_key_count, - files.current_chunks().unwrap_or(0) - ); - state.phase = MigrationPhase::Bridging; - state.committed_at_unix = None; - state.rebuilds_since_commit = 0; - if let Err(e) = state.save(&config.root_dir) { - warn!("Could not persist the migration marker: {e}"); - } - } else if state.phase == MigrationPhase::Committed && legacy_only > state.shed_key_count { - // Once committed, the copier only moves keys the rank check refuses to shed. - // A chunk that turns up in the legacy environment after that, and that this - // node never agreed to give up, is never copied and never proven to be held - // elsewhere, so nothing frees it from the environment and the environment is - // never retired. Free disk does not change that; only the phase does. - // - // The count, not emptiness, is the test. A node that legitimately shed keeps - // exactly the keys it is giving up in the legacy environment until they stop - // being answerable, and sending it back to the bridge on every restart would - // reset its retention clock each time. - warn!( - migration_event = "back_to_bridging", - legacy_only, - shed = state.shed_key_count, - "The migration marker says this node committed after agreeing to shed {} \ - chunk(s), but {legacy_only} are still only in the legacy environment. \ - Restarting the migration from the copying stage so they are copied.", - state.shed_key_count - ); - state.phase = MigrationPhase::Bridging; - state.committed_at_unix = None; - state.rebuilds_since_commit = 0; - if let Err(e) = state.save(&config.root_dir) { - warn!("Could not persist the migration marker: {e}"); - } - } else if legacy.is_some() && state.phase == MigrationPhase::FilesOnly { - warn!( - "The migration marker says this node is done but {} is still on disk. \ - Resuming the bridge.", - legacy_env_dir.display() - ); - state.phase = MigrationPhase::Bridging; - if let Err(e) = state.save(&config.root_dir) { - warn!("Could not persist the migration marker: {e}"); - } + self.measure(snapshot)?; + if snapshot.free_estimate() < want { + // Do not cache a failing result: `measured_at` is left set so the next call + // still re-measures, because the branch above only short-circuits a pass. + return Err(Error::Storage(format!( + "Insufficient disk space: {:.2} GiB available, {:.2} GiB reserve required. \ + Free disk space or increase the partition to continue storing chunks.", + bytes_to_gib(snapshot.free_estimate()), + bytes_to_gib(self.reserve), + ))); } + Ok(()) + } - let store = Self { - files, - legacy: parking_lot::RwLock::new(legacy), - retirement: tokio::sync::RwLock::new(()), - legacy_env_dir, - config, - state: parking_lot::RwLock::new(state), - key_locks: std::iter::repeat_with(|| tokio::sync::Mutex::new(())) - .take(KEY_LOCK_LANES) - .collect(), - }; + /// Drop the cached measurement so the next question hits the filesystem. + fn invalidate(&self) { + let mut snapshot = self.snapshot.lock(); + snapshot.measured_at = None; + snapshot.written_since = 0; + } - let (file_keys, legacy_keys) = store.split_counts(); - info!( - "Chunk store ready: {file_keys} chunks in files, {legacy_keys} still only in the \ - legacy environment, phase {:?}", - store.migration_phase() - ); - Ok(store) + /// Return `Ok(())` if a write of `needed` bytes would fit. Charges nothing. + fn check(&self, needed: u64) -> Result<()> { + let mut snapshot = self.snapshot.lock(); + self.admit(&mut snapshot, needed) } - /// Open the legacy environment and work out which keys only it holds. - async fn open_legacy(config: &ChunkStoreConfig, files: &FileStore) -> Result { - let (lmdb, legacy_keys) = Self::open_legacy_env(config).await?; - Ok(Legacy { - lmdb, - only: Arc::new(parking_lot::RwLock::new(Self::keys_only_in_legacy( - &legacy_keys, - files, - ))), - skipped_rollback_copies: Arc::new(std::sync::atomic::AtomicU64::new(0)), - pending: Arc::new(parking_lot::RwLock::new(BTreeMap::new())), + /// Admit a write of `needed` bytes and charge it in the same critical section. + /// + /// Checking and charging separately is the bug this exists to prevent: dozens of + /// protocol handlers can each pass against the same cached measurement before any of + /// them has written a byte, and collectively cross the reserve. + /// + /// The returned [`Reservation`] settles itself when dropped, so a caller whose future + /// is dropped mid-write cannot strand it. Nothing else ever decrements the in-flight + /// count, so a stranded reservation would be permanent, and enough of them would make + /// an empty disk look full until the process restarted. + fn reserve(self: &Arc, needed: u64) -> Result { + { + let mut snapshot = self.snapshot.lock(); + self.admit(&mut snapshot, needed)?; + snapshot.in_flight = snapshot.in_flight.saturating_add(Self::charge(needed)); + } + Ok(Reservation { + capacity: Arc::clone(self), + bytes: needed, + settled: false, }) } - /// Open the legacy environment and read every key in it. - /// - /// Split from the diff against the file store because the two want different timing: - /// this is slow and safe to do at any moment, the diff has to be the last thing before - /// the handle is installed. - async fn open_legacy_env( - config: &ChunkStoreConfig, - ) -> Result<(Arc, Vec)> { - let lmdb = Arc::new( - LmdbStorage::new(LmdbStorageConfig { - root_dir: config.root_dir.clone(), - verify_on_read: config.verify_on_read, - max_map_size: config.max_map_size, - disk_reserve: config.disk_reserve, - }) - .await?, - ); - // From here it never grows. Two stores on one disk each measure the same free - // space and neither knows what the other is about to spend, so a chunk written to - // both can be admitted twice against one lot of headroom and the pair can cross - // the reserve together. Pinned, this one writes only from pages it already has, - // so the file store's accounting is the only claim on free disk. - // - // What it costs is that the rollback copy is made only when the environment has - // room of its own. That is the right way round: the copy exists to make a fleet - // rollback survivable, not to be the write that has to succeed, and an - // environment this migration exists to delete should not be taking new disk to - // hold a second copy of something the file store already has. - lmdb.pin_growth().await?; - let legacy_keys = lmdb.all_keys().await?; - Ok((lmdb, legacy_keys)) + /// Give back a reservation whose write did not happen. + fn release(&self, needed: u64) { + let mut snapshot = self.snapshot.lock(); + snapshot.in_flight = snapshot.in_flight.saturating_sub(Self::charge(needed)); } - /// Which of `legacy_keys` the file store does not have. - /// - /// In memory, no I/O: the file store answers from its index. Cheap enough to redo - /// immediately before installing a handle, which is the point. A key that lost its - /// file while the environment was being read must be in this set, or nothing will - /// look for it again and retirement will destroy the copy that is left. - fn keys_only_in_legacy(legacy_keys: &[XorName], files: &FileStore) -> BTreeSet { - legacy_keys - .iter() - .filter(|key| !files.is_indexed(key)) - .copied() - .collect() + /// Turn a reservation into bytes that are now on disk. + fn commit_reservation(&self, needed: u64) { + let charge = Self::charge(needed); + let mut snapshot = self.snapshot.lock(); + snapshot.in_flight = snapshot.in_flight.saturating_sub(charge); + snapshot.written_since = snapshot.written_since.saturating_add(charge); } - /// How many keys only the legacy environment holds; zero once it is gone. - fn legacy_only_count(legacy: Option<&Legacy>) -> u64 { - legacy.map_or(0, |l| l.only.read().len().try_into().unwrap_or(u64::MAX)) + /// Credit a completed delete back to the cached measurement. + fn record_removed(&self, len: u64) { + let mut snapshot = self.snapshot.lock(); + snapshot.written_since = snapshot.written_since.saturating_sub(Self::charge(len)); } - /// Take the critical section for one key. - async fn key_lock(&self, address: &XorName) -> Option> { - let lane = address.last().copied().unwrap_or(0) as usize; - match self.key_locks.get(lane) { - Some(lock) => Some(lock.lock().await), - None => None, + /// The two accounting counters, for tests that need to prove a charge was + /// settled one way rather than the other. + #[cfg(test)] + fn counters(&self) -> (u64, u64) { + let snapshot = self.snapshot.lock(); + (snapshot.written_since, snapshot.in_flight) + } +} + +/// A charged, unsettled write. +/// +/// Held by whatever is actually doing the write, so the charge is released even if the +/// caller's future is dropped and only the blocking closure survives. +pub struct Reservation { + /// The guard this was taken from. + capacity: Arc, + /// Payload size, before rounding. + bytes: u64, + /// Whether it has already been accounted for. + settled: bool, +} + +impl Reservation { + /// The write landed: move the charge from in-flight to written. + pub(crate) fn commit(mut self) { + self.capacity.commit_reservation(self.bytes); + self.settled = true; + } +} + +impl Drop for Reservation { + fn drop(&mut self) { + if !self.settled { + self.capacity.release(self.bytes); } } +} + +/// Environment variable naming a failpoint: stop after the temp file, before the rename. +#[cfg(any(test, feature = "test-utils"))] +pub const HALT_BEFORE_PUBLISH: &str = "ANT_HALT_BEFORE_PUBLISH"; + +/// Park forever at a named failpoint, once a marker says the process has reached it. +/// +/// For crash tests, which need a process to die *inside* an operation rather than at +/// whatever point a sleep in another process happened to land. The variable holds a path: +/// this writes it, so the parent knows the child is exactly here, and then waits to be +/// killed. +/// +/// Costs one environment read per write when the feature is compiled in, and the feature +/// is not in a release build. +#[cfg(any(test, feature = "test-utils"))] +pub(crate) fn halt_here_if_asked(variable: &str, reached: &Path) { + let Ok(marker) = std::env::var(variable) else { + return; + }; + // Let the first few through. A test that stops the very first write leaves a store + // with nothing successfully in it, and an assertion over what it holds then passes by + // iterating nothing. Letting some land first means the crash happens to a store that + // has real chunks in it, which is the situation worth checking. + let skip: u64 = std::env::var(HALT_AFTER) + .ok() + .and_then(|raw| raw.parse().ok()) + .unwrap_or(0); + if HALTS_SEEN.fetch_add(1, std::sync::atomic::Ordering::AcqRel) < skip { + return; + } + if let Err(e) = std::fs::write(&marker, reached.as_os_str().as_encoded_bytes()) { + // The parent waits for this file. Saying so on the way past is the difference + // between a test that fails and one that hangs until the job times out. + eprintln!("failpoint could not write its marker {marker}: {e}"); + return; + } + loop { + std::thread::sleep(Duration::from_hours(1)); + } +} + +/// How many writes to let through before the failpoint fires. +#[cfg(any(test, feature = "test-utils"))] +pub const HALT_AFTER: &str = "ANT_HALT_AFTER"; + +/// How many times the failpoint has been reached in this process. +#[cfg(any(test, feature = "test-utils"))] +static HALTS_SEEN: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); - /// A cheap clone of the legacy handle, or `None` once it is retired. - fn legacy(&self) -> Option { - self.legacy.read().clone() +/// Clears a write's registration when the work finishes, however it finishes. +/// +/// Held by the blocking closure rather than by the caller, so a dropped future cannot +/// leave an entry behind, and a panic in the work cannot either. +struct WriteInFlight { + writing: Arc>>, + finished: Arc, + address: XorName, +} + +impl Drop for WriteInFlight { + fn drop(&mut self) { + let was_last = { + let mut writing = self.writing.lock(); + match writing.get_mut(&self.address) { + Some(count) if *count > 1 => { + *count -= 1; + false + } + _ => { + writing.remove(&self.address); + true + } + } + }; + // Only when this was the last one. Waking a waiter while another write for the + // same key is still queued is exactly what the count exists to prevent. + if was_last { + self.finished.notify_waiters(); + } } +} + +/// What is behind a chunk's name on disk. +/// +/// Four answers, not two, because "could not read it" must never be treated as "wrong": +/// replacing a chunk is destructive, and off Unix it truncates the file in place, so a +/// transient fault would turn a healthy sole copy into an empty one. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum StoredBytes { + /// The bytes are there and hash to the name. + Good, + /// The bytes are there and do not. + Wrong, + /// There is nothing behind the name. + Absent, + /// The question could not be answered this time. + Unreadable, +} + +/// Convert a byte count to GiB for human-readable log messages. +#[allow(clippy::cast_precision_loss)] // display only — sub-byte precision is irrelevant +fn bytes_to_gib(bytes: u64) -> f64 { + bytes as f64 / (1024.0 * 1024.0 * 1024.0) +} + +/// Content-addressed store holding one immutable file per chunk. +/// +/// The filesystem is the sole authority. The in-memory index is a cache of what the +/// directory tree already contains, rebuilt from directory entries at every open, and +/// every mutation of it mirrors a filesystem operation that has *already* completed. +/// Bitcask's issue #114 is the cautionary tale for the opposite order: an index that is +/// rebuilt at startup and then mutated in anticipation drifts, and the drift is silent. +#[derive(Debug)] +pub struct ChunkStore { + /// Store configuration. + config: ChunkStoreConfig, + /// `{root_dir}/chunks`. + chunks_dir: PathBuf, + /// Every address whose file is published, in ascending order. + /// + /// `BTreeSet` rather than a hash set because `all_keys()` must be sorted (the + /// commitment builder truncates with `take(cap)` *before* the Merkle tree sorts, so + /// an unstable order would make the node's published commitment depend on iteration + /// luck), and because it never spikes memory while growing. + index: Arc>>, + /// One mutex per shard, serialising writers of the same address. + /// + /// LMDB gave exactly-once `put` semantics for free: the duplicate test happened + /// inside the write transaction. Two threads publishing the same address here would + /// otherwise both see an absent file, both rename, and both report "newly stored", + /// double-counting the chunk. The lane is indexed by the address's LAST byte for the + /// same reason the shard is: a node's keys share their leading bytes, so lanes keyed + /// on the first byte would all collapse into one. + write_lanes: Arc>>, + + /// One lock per shard, held across a whole logical transition for a key. + /// + /// Not the same thing as the write lanes above, which are taken inside a blocking + /// closure and make one file write atomic. These are held across await points, which is + /// what the races that matter need: a delete has to exclude a read that is deciding + /// whether to accept an offered copy, and both span an await. Without it a prune can + /// remove the file between that read and its answer, and the caller is told the chunk + /// is already held while the copy that would have replaced it is discarded. + /// + /// Indexed by the address's LAST byte, for the reason the shard is: a node's keys share + /// their leading bytes, so lanes keyed on the first would collapse into one. + key_locks: Arc>>, + /// Operation counters, same shape as the LMDB store reported. + stats: parking_lot::RwLock, + /// Which of the 256 shard directories are known to exist, so a steady-state write + /// does not pay a `create_dir_all` syscall. + shards_present: Arc>, + /// Indexed chunks this store currently cannot read. + /// + /// Held back from everything the node says it has, while the files themselves are + /// left alone. See [`Self::mark_suspect`]. + suspect: Arc>>, + /// Indexed chunks a read has proven do not match their name. + /// + /// Separate from the above because they clear differently. Not being able to read a + /// file is a question a later read answers; bytes that are wrong stay wrong however + /// often they are read, and only a repair or a removal settles it. A raw read that + /// does not hash anything must not take a chunk out of this set. + known_wrong: Arc>>, + /// Addresses this store is part-way through writing. + /// + /// Every mutation registers here before it spawns its blocking work and clears the + /// entry *inside* that work, so a caller whose future is dropped cannot skip the + /// clearing while the write itself goes on to land. That is the difference that + /// matters: the blocking half is not cancelled with the future, so anything the + /// future was going to do afterwards is not a record of what happened. + /// + /// It lets a delete queue behind the exact write it would otherwise race, rather than + /// behind every write this store has in flight. + /// + /// Counted, not a set. Cancellation can release the facade's key lane while the + /// blocking half survives, so a second write for the same key can start behind the + /// first. With one entry between them, whichever finished first would remove it and a + /// waiter would be told the key is free while the other was still queued. + writing: Arc>>, + /// Woken when [`Self::writing`] loses its last entry for a key. + write_finished: Arc, + /// Size-aware free-space predicate. + capacity: Arc, + /// Monotonic counter that makes temp filenames unique within this store. + temp_seq: AtomicU64, + /// Random per-instance discriminator for temp filenames. + nonce: u32, + /// Held for the store's lifetime. Startup fails without it. + /// + /// Shared rather than owned so the blocking work that depends on it can hold a lease + /// of its own: that work outlives the future that spawned it, and a cancelled caller + /// releasing the lock would leave it writing into a directory another process had + /// just been let into. + lock: Arc, + /// Tracks every blocking task, so [`ChunkStore::wait_idle`] can wait for writes that + /// outlived their awaiting future. + blocking_tracker: TaskTracker, + /// Test-only gate read-acquired at the top of the put blocking closure. + /// + /// Tests hold the write half to park an in-flight write on the blocking pool, which + /// is the shape a `select!` losing to a shutdown token leaves behind. + /// + /// Compiled under `test-utils` too, unlike the gate below, because the browser devnet + /// harness parks a real write through it. What that costs a write in a `test-utils` + /// build is an uncontended read taken on the blocking thread, not an await. + #[cfg(any(test, feature = "test-utils"))] + test_put_gate: Arc>, + + /// Test-only: parks a put after it has taken the key's lane and before it registers + /// itself as in flight. + /// + /// Asynchronous, unlike the gate above. That one is taken inside a blocking closure on + /// its own thread; this one is taken on the runtime, so a synchronous lock here would + /// block the executor and the test would deadlock instead of observing anything. + /// + /// A separate gate from the one above, because that one sits inside the blocking + /// closure, which is after registration. The window this opens is the one the key lane + /// exists for: a put that a delete's wait cannot see yet, because there is nothing to + /// see. Without a hook here, a test cannot tell a delete blocked by the lane from a + /// delete blocked by the wait, and so cannot show the lane is doing anything. + #[cfg(test)] + test_pre_registration_gate: Arc>, + + /// Test-only: how many puts have reached that gate. + /// + /// So a test can wait for the put to be parked rather than sleeping and hoping. A sleep + /// makes the staging a guess, and a guess in a test that is meant to be deterministic + /// is a flake waiting for a loaded machine. + #[cfg(test)] + test_reached_pre_registration: Arc, +} + +impl ChunkStore { + /// Open (or create) the store at `{root_dir}/chunks/`. + /// + /// Sweeps orphaned temp files, then rebuilds the index from directory entries. + /// The scan reads names only: it never `stat`s an entry and never reads a chunk. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] if the directory cannot be created, the layout marker + /// is unreadable or describes a layout this build does not implement, or the scan + /// fails. + pub async fn new(config: ChunkStoreConfig) -> Result { + // Deliberately nothing about the old chunk store here. Clearing up after the storage + // migration happens once, in `NodeBuilder::build`, and only after this constructor + // has succeeded: what it deletes are directories whose chunks are in this store, so + // it may not run until this store is open. Asking here as well would put that + // decision in front of itself. + + let chunks_dir = config.root_dir.join(CHUNKS_DIR_NAME); + std::fs::create_dir_all(&chunks_dir).map_err(|e| { + Error::Storage(format!( + "Failed to create chunk store directory {}: {e}", + chunks_dir.display() + )) + })?; + + check_path_budget(&chunks_dir); + + let layout = read_or_write_layout(&chunks_dir)?; + layout.check_supported()?; + + // Startup fails without it, so from here this process is the only one using this + // directory and an interrupted write can only be its own. + let lock = acquire_store_lock(&chunks_dir)?; + + let scan_dir = chunks_dir.clone(); + // The scan holds the lease itself. It sweeps interrupted writes on the strength of + // being alone here, and it runs on a thread that outlives this future: a + // cancelled startup that released the lock would leave it sweeping a directory + // another process had just been let into. + let scan_lease = Arc::clone(&lock); + // The node root as well as the chunk tree. The scan sweeps interrupted writes + // under `chunks/`, which covers the layout marker's temporary because that lives + // there; the migration marker's lives in the root, where nothing looked. + let root = config.root_dir.clone(); + let scan = spawn_blocking(move || { + let _lease = scan_lease; + sweep_marker_temps(&root); + scan_store(&scan_dir) + }) + .await + .map_err(|e| Error::Storage(format!("Chunk store scan task failed: {e}")))??; + + let ScanResult { + keys, + shards_present, + swept_temps, + skipped, + } = scan; + + let key_count = keys.len(); + // Build from a sorted vector: bulk-building packs every B-tree node to its + // capacity, where repeated `insert` converges on ~68% fill for the same keys. + let index: BTreeSet = keys.into_iter().collect(); + + if swept_temps > 0 { + info!("Chunk store: removed {swept_temps} orphaned temporary file(s) from interrupted writes"); + } + if skipped > 0 { + warn!("Chunk store: ignored {skipped} directory entr(ies) that are not chunk files"); + } + info!( + "Chunk store open at {} ({key_count} chunks)", + chunks_dir.display() + ); + + let capacity = Arc::new(CapacityGuard::new(chunks_dir.clone(), config.disk_reserve)); - /// `(chunks in files, chunks only in the legacy environment)`. - fn split_counts(&self) -> (u64, u64) { - // Legacy first, for the reason given on `exists`: a key mid-copy is then counted - // twice for an instant rather than not at all, and over-reporting what the node - // holds is the safe direction for every caller of `current_chunks`. - let legacy = self - .legacy() - .map_or(0, |l| l.only.read().len().try_into().unwrap_or(u64::MAX)); - let files = self.files.current_chunks().unwrap_or(0); - (files, legacy) + Ok(Self { + config, + chunks_dir, + index: Arc::new(parking_lot::RwLock::new(index)), + key_locks: Arc::new( + std::iter::repeat_with(|| tokio::sync::Mutex::new(())) + .take(SHARD_COUNT) + .collect(), + ), + write_lanes: Arc::new( + std::iter::repeat_with(|| parking_lot::Mutex::new(())) + .take(SHARD_COUNT) + .collect(), + ), + stats: parking_lot::RwLock::new(StorageStats::default()), + shards_present: Arc::new(parking_lot::Mutex::new(shards_present)), + suspect: Arc::new(parking_lot::RwLock::new(HashSet::new())), + known_wrong: Arc::new(parking_lot::RwLock::new(HashSet::new())), + writing: Arc::new(parking_lot::Mutex::new(HashMap::new())), + write_finished: Arc::new(tokio::sync::Notify::new()), + capacity, + temp_seq: AtomicU64::new(0), + nonce: rand::random(), + lock, + blocking_tracker: TaskTracker::new(), + #[cfg(any(test, feature = "test-utils"))] + test_put_gate: Arc::new(parking_lot::RwLock::new(())), + #[cfg(test)] + test_pre_registration_gate: Arc::new(tokio::sync::RwLock::new(())), + #[cfg(test)] + test_reached_pre_registration: Arc::new(std::sync::atomic::AtomicU64::new(0)), + }) } /// Store a chunk. /// - /// While a legacy environment exists and dual-writing is on, the chunk goes there - /// **first**. A chunk uploaded during the bridge to holders that all revert to a - /// pre-migration build would otherwise be gone from every one of them, and that is - /// real client data, not a replica. + /// On Unix, publishing is a rename within the destination directory, so the final name + /// can never appear on partial content: the name *is* the hash, and the content is + /// fully written and flushed before the name exists. Off Unix there is no rename, for + /// the reason `publish_in_place` gives (it is compiled only on those platforms, so this + /// is not a link), and a partial file can wear a real name; that + /// is why a duplicate is read and compared rather than trusted. /// /// # Returns /// - /// `true` if the chunk was newly stored, `false` if either store already had it. + /// `true` if the chunk was newly stored, `false` if it was already present. /// /// # Errors /// - /// Returns [`Error::Storage`] if the content does not hash to `address`, the disk is - /// too full, or the write fails. + /// Returns [`Error::Storage`] if the content does not hash to `address`, the disk + /// is too full, or the write fails. pub async fn put(&self, address: &XorName, content: &[u8]) -> Result { - // Shared, for the reason given on the field: retirement waits for the legacy - // environment to go idle, and a write that keeps starting new work in it while - // that wait runs makes the wait unbounded. It also stops a write inserting a key - // into the legacy-only set after the gates have approved the set that may go. - let _using_legacy = self.retirement.read().await; + // The key's whole transition, not just the part that touches the disk. Registering + // the write is what a delete waits for, and everything before that registration + // happens outside it: validation, the duplicate read, the capacity reservation. A + // put that got that far before a delete arrived would otherwise be invisible to the + // delete's wait, register while the delete was already committed to going ahead, + // and publish afterwards. The node would then hold a chunk it had decided to prune. let _lane = self.key_lock(address).await; - let legacy = self.legacy(); - let already_in_legacy = legacy - .as_ref() - .is_some_and(|l| l.only.read().contains(address)); - - let mut dual_written = false; - if let Some(ref l) = legacy { - if self.config.migration.dual_write_legacy && !already_in_legacy { - // The legacy store's own verdict, not the file store's. It accounts for - // pages it can reuse internally, which is the right question for a write - // into it and the wrong one for the file about to be written. A full - // legacy store must not fail a put the file store can serve: the copy is - // there to make a fleet rollback survivable, and losing that for one - // chunk is much better than refusing the chunk. - if l.lmdb.capacity_verdict() == crate::storage::CapacityVerdict::Full { - // Counted here as well as on the failure path below, and this is the - // one that matters: a pinned environment with no reusable page answers - // Full for every chunk, so on the node most affected this is the whole - // of the skipping and the other path never runs at all. - if let Some(count) = l.note_skipped_rollback_copy() { - warn!( - migration_event = "no_rollback_copy", - skipped = count, - "Legacy chunk environment is full; storing {} in files only. A \ - rollback to a pre-migration build would not have this chunk, \ - and {count} write(s) on this node have now gone without a \ - rollback copy.", - hex::encode(address) - ); - } else { - debug!( - "Legacy chunk environment is full; storing {} in files only.", - hex::encode(address) - ); - } - } else { - // Announced BEFORE the write, not after it. The write runs on a - // blocking thread that outlives this future: a shutdown that drops the - // caller mid-way can leave the environment holding a chunk while - // nothing here ever ran to record it, and a key in neither view is - // what retirement destroys. In the in-flight note rather than the key - // set, because until the write returns this node does not hold the - // chunk and must not say it does. - l.announce(address); - // Best effort, and only best effort. The verdict above is optimistic - // by design: LMDB can still refuse a write for fragmentation, pages - // pinned by a long read, or a copy-on-write B-tree split. Propagating - // that would let a store this node is in the middle of abandoning - // reject paid chunks the file store has ample room for, for the whole - // bridge period. The chunk's own validity is not at stake here; the - // file store checks the content address itself. - match l.lmdb.put(address, content).await { - Ok(_) => dual_written = true, - Err(e) => { - if let Some(count) = l.note_skipped_rollback_copy() { - warn!( - migration_event = "no_rollback_copy", - skipped = count, - "Could not also write {} to the legacy environment: \ - {e}. Storing it in files only. A rollback to a \ - pre-migration build would not have this chunk, and \ - {count} write(s) on this node have now gone without a \ - rollback copy.", - hex::encode(address) - ); - } - } - } - } + let computed = crate::client::compute_address(content); + if computed != *address { + return Err(Error::Storage(format!( + "Content address mismatch: expected {}, computed {}", + hex::encode(address), + hex::encode(computed) + ))); + } + // The read path refuses anything over the ceiling, so writing one would create a + // file the store could never read back and could never repair. + if content.len() > MAX_CHUNK_SIZE { + return Err(Error::Storage(format!( + "Chunk {} is {} bytes, over the {MAX_CHUNK_SIZE} byte maximum", + hex::encode(address), + content.len() + ))); + } + + // An indexed name is not proof of the bytes under it. The index is built from + // names, by the startup scan and by a completed publish, and a name can outlive + // what it points at: off Unix a chunk is created under its final name before its + // bytes are written, so a crash leaves a short file wearing a real name, and rot + // leaves a full-length one. Answering "already have it" to the copy that would fix + // either is how a node discards its own repair and is never offered another. + // + // So the bytes decide. Checked before the reservation below, so re-storing a chunk + // this node already holds stays a no-op on a full disk. + if self.index.read().contains(address) { + if let Some(answer) = self.settle_indexed_duplicate(address, content).await { + return answer; } } - let stored_in_files = match self.files.put(address, content).await { - Ok(stored) => stored, - Err(e) => { - // The bytes reached LMDB but not the file store. Record the key as - // legacy-only so the union still finds it and the copier retries later; - // without this the node would hold a chunk it could not serve. + let len = content.len() as u64; + // Reserved after the duplicate test so re-storing an existing chunk stays a + // harmless no-op on a full disk, matching the LMDB store's ordering. + let reservation = self.capacity.reserve(len)?; + + let shard = self.chunks_dir.join(shard_name(address)); + let final_path = shard.join(hex::encode(address)); + let temp_path = shard.join(self.next_temp_name()); + let payload = content.to_vec(); + let lanes = Arc::clone(&self.write_lanes); + let index = Arc::clone(&self.index); + let shards_present = Arc::clone(&self.shards_present); + let chunks_dir = self.chunks_dir.clone(); + let lane = shard_index(address); + let key = *address; + #[cfg(any(test, feature = "test-utils"))] + let test_put_gate = Arc::clone(&self.test_put_gate); + // Registered before the work is spawned and cleared by the work itself, so a + // caller that goes away cannot leave a delete free to race this publish. + // Test-only: the window between taking the lane and being visible to a delete. + #[cfg(test)] + { + self.test_reached_pre_registration + .fetch_add(1, std::sync::atomic::Ordering::AcqRel); + drop(self.test_pre_registration_gate.read().await); + } + let in_flight = self.begin_write(address); + // And the lease, for the same reason the scan holds it: this thread writes into a + // directory whose exclusivity the lock is what establishes, and it can outlive + // the last owner of the store. + let lease = Arc::clone(&self.lock); + let known_wrong = Arc::clone(&self.known_wrong); + let suspect = Arc::clone(&self.suspect); + + let outcome = self + .blocking_tracker + .spawn_blocking(move || -> Result { + let _in_flight = in_flight; + let _lease = lease; + // Test-only: parks here while a test holds the write half. + #[cfg(any(test, feature = "test-utils"))] + let _test_put_gate = test_put_gate.read(); + let _lane = lanes.get(lane).map(parking_lot::Mutex::lock); + // `mkdir` plus a directory flush are syscalls, so they belong here and + // not on a runtime worker. + ensure_shard_dir(&chunks_dir, &shard, lane, &shards_present)?; + let outcome = match publish(&temp_path, &final_path, &payload, &shard) { + Ok(outcome) => outcome, + Err(PublishFailed { error, left_behind }) => { + // A publish that failed can still have left the bytes there: off + // Unix the chunk is created under its final name, and if the write + // or the flush then fails, the cleanup that removes it can fail + // too. Releasing the reservation would hand back a charge for a + // file that is on the disk. + // + // The publish says so rather than this deciding from a later + // `is_file`. Asking the filesystem afterwards infers ownership from + // a name being occupied, which is true under the store lock and the + // shard lane and not true against anything out of band, and this + // file spends a lot of its length arguing that a name is not + // evidence. A bit set by the code that created the file is. + if left_behind { + reservation.commit(); + } + return Err(error); + } + }; + // Placed, not yet durable. A failure from here on leaves the bytes on the + // disk: the chunk is rightly not reported as stored, because a copy that is + // not durable must not authorise deleting another, but the space is spent + // all the same. Dropping the reservation would hand that charge back and + // admit the next write against room that is already gone. // - // Only when the file store really does not have it. A write can fail - // because the file that is already there could not be read to check it, - // and calling that key legacy-only while the file index still names it - // puts it in both views, where it stays answerable and vetoes retirement - // for good. - // The file half failed and the legacy half did not, so the environment - // holds the only copy and the key really is legacy-only now. Promoted - // from the in-flight note to the key set, which is the one moment that - // promotion is warranted: both outcomes are known. - if let Some(ref l) = legacy { - l.announced_write_finished(address); - if dual_written && !self.files.is_indexed(address) { - l.only.write().insert(*address); + // Only for a chunk this call published. `Duplicate` means the file was + // already there and was charged by whoever wrote it, so charging it again + // here would count one file twice and shrink the store's idea of its own + // disk on every retry. + if let Err(e) = flush_publication(&final_path, &shard) { + if matches!(outcome, PutOutcome::New) { + reservation.commit(); } + return Err(e); } - return Err(e); + // Index inside the lane, and only after the rename has returned. A + // concurrent delete of the same address therefore cannot interleave + // between publishing the file and admitting the key. + // + // Only for a chunk this call actually published. `Duplicate` says a file + // already wears the name, and a name is not evidence about the bytes under + // it: the four-way answer that decides whether they are good, wrong, absent + // or unreadable runs after the await below, and a caller whose future is + // dropped never reaches it. Admitting the key here would leave the node + // claiming, advertising and committing to bytes nothing has read, with no + // suspect or known-wrong mark to hold it back, and the sharpest case is a + // name the startup scan deliberately refused because what wears it is a + // fifo, a socket or a directory. The duplicate arm admits the key itself, + // once a read has proven the bytes. + if matches!(outcome, PutOutcome::New) { + index.write().insert(key); + // With the marks that would otherwise hold the key back. These bytes + // were hashed against their own name on the way in, so an older + // instance proven wrong or merely unreadable has just been replaced by + // a good one. Cleared here rather than after the await for the same + // reason the insert is here: a cancelled caller would leave the key + // indexed and suppressed at once, so a chunk this node really does hold + // would stay hidden from `exists` and `all_keys` until some later read + // happened to settle it. + known_wrong.write().remove(&key); + suspect.write().remove(&key); + // Settled here, inside the work, so a dropped awaiter cannot strand it. + reservation.commit(); + } + Ok(outcome) + }) + .await + .map_err(|e| Error::Storage(format!("Chunk store put task failed: {e}")))??; + + match outcome { + PutOutcome::Duplicate => self.settle_duplicate(address, content).await, + PutOutcome::New => { + // Freshly published bytes that were checked against their own name on the + // way in. The marks were already cleared inside the work, where a dropped + // caller cannot skip them; what is left here is only what a caller who is + // still waiting should see. + let mut stats = self.stats.write(); + stats.chunks_stored = stats.chunks_stored.saturating_add(1); + stats.bytes_stored = stats.bytes_stored.saturating_add(len); + drop(stats); + debug!("Stored chunk {} ({len} bytes)", hex::encode(address)); + Ok(true) } - }; - - // The file store has it, so it is not legacy-only, whether it was already there - // or this call put it there. The in-flight note goes at the same time: both - // writes have returned, so there is nothing left in flight to protect. - if let Some(ref l) = legacy { - l.only.write().remove(address); - l.announced_write_finished(address); } - if already_in_legacy { - // Migrated for free: a hot key the copier no longer has to move. - return Ok(false); - } - Ok(stored_in_files) } - /// Retrieve a chunk, verifying it against its address when configured to. + /// Decide what a name that was already taken actually means. + /// + /// Split out of [`Self::put`] because it is a different question. `put` puts bytes on + /// a disk; this reads bytes back to find out whether the ones already there are the + /// ones the caller is offering, which is the only thing that makes a duplicate safe to + /// report as stored. /// /// # Errors /// - /// Returns [`Error::Storage`] on an I/O failure, or when verification fails and no - /// intact copy is available. - pub async fn get(&self, address: &XorName) -> Result>> { - // Held for the whole read. A verifying read that finds rotted bytes throws the - // file away, and until this key is back in the legacy-only set there is a moment - // when it appears to live in neither store. Retirement waits behind this rather - // than deleting the copy the read is about to fall back on. - let _reading = self.retirement.read().await; - let fallback = self.legacy(); - match self.files.get(address).await { - Ok(Some(content)) => Ok(Some(content)), - Ok(None) => { - // The file store missed. If the legacy store answers, the key has to go - // back into the union view: the file index has just dropped it, and a key - // in neither view is skipped by the verification pass and destroyed by - // retirement. - self.serve_from_legacy(address, fallback).await + /// Returns [`Error::Storage`] when the existing file is absent or could not be read, + /// both of which mean this node must not report the chunk as held. + async fn settle_duplicate(&self, address: &XorName, content: &[u8]) -> Result { + // The file was already on disk, and its name is not evidence its contents + // are right. The startup scan indexes by name without reading anything, + // and on Windows a crash mid-write leaves a partial file under a real + // chunk name. Trusting the name here would acknowledge a chunk that was + // never stored, and then discard the good copy arriving to repair it. + // Every answer handled, because three of the four must not report the + // chunk as stored. A caller that hears success acts on it: a client drops + // its own copy, replication marks the key held, and the copier takes it + // out of the legacy-only set. + match self.stored_bytes_match(address).await { + StoredBytes::Good => { + // Admitted here, which is the first moment the bytes behind the + // name have been read and shown to hash to it. Idempotent: the + // ordinary case is a key the startup scan already indexed. + self.index.write().insert(*address); + { + let mut stats = self.stats.write(); + stats.duplicates = stats.duplicates.saturating_add(1); + } + Ok(false) } - Err(e) => { - // Whatever went wrong with the file, the legacy environment may still - // have the bytes, and while it is there it is the point of the bridge to - // use them. A verification failure means the file was thrown away; every - // other error (a full descriptor table, an I/O fault, an oversized file) - // leaves the file in place and unreadable. Both are unservable from the - // file store, and both are worth asking the other store about. - let verification_failed = format!("{e}").contains("verification failed"); + StoredBytes::Wrong => { warn!( - "Chunk {} could not be served from the file store ({e}); looking for a \ - copy in the legacy environment", + "Chunk {} was already on disk but its contents are wrong; \ + replacing it with the copy just offered", hex::encode(address) ); - let from_legacy = self.serve_from_legacy(address, fallback).await; - match from_legacy { - Ok(Some(content)) => Ok(Some(content)), - // Nothing anywhere. Report the original failure rather than a plain - // miss, so the caller can tell the difference. The key is only - // re-queued for copying when the file really went: an unreadable file - // that is still there is not legacy-only, and calling it so is how a - // key ends up claimed through one view and servable through neither. - Ok(None) => Err(e), - Err(legacy_error) => { - if verification_failed { - Err(e) - } else { - Err(legacy_error) - } - } - } + self.repair_holding_the_lane(address, content) + .await + .map(|()| true) } + // The name was taken a moment ago and is not now, or was never a + // readable chunk file. Either way nothing holds these bytes, so say so + // rather than reporting a chunk that is not there. + StoredBytes::Absent => Err(Error::Storage(format!( + "Chunk {} was reported already on disk but nothing is there. Not \ + reporting it as stored.", + hex::encode(address) + ))), + // Replacing on an unanswered question would destroy a healthy copy, + // and reporting success would discard the offered one. The index entry + // stays: the file is still there, and dropping the entry would leave + // the chunk in neither this store's view nor the legacy one, which is + // what retirement destroys. Removing an entry is the quarantine path's + // job, and it removes the file with it, after a read that succeeded + // and proved the bytes wrong. + StoredBytes::Unreadable => Err(Error::Storage(format!( + "Chunk {} is on disk but could not be read to check it. Not \ + replacing it, and not reporting it as stored.", + hex::encode(address) + ))), } } - /// Serve a key the file store could not, from the legacy store, and put it back on - /// the copier's list. + /// Take the critical section for one key. /// - /// The whole sequence runs under the key's critical section, including the legacy - /// read. Reading first and locking afterwards would let a concurrent delete remove - /// both backings in between, and the key would then be re-inserted from bytes that no - /// longer exist anywhere: a phantom entry that `exists` reports and `get` never - /// satisfies. - async fn serve_from_legacy( - &self, - address: &XorName, - legacy: Option, - ) -> Result>> { - // The handle the caller took before it read the file. Not re-fetched here: the - // point of taking it early is that it has been held continuously since before the - // file could be thrown away, so retirement cannot have run in between. - let Some(legacy) = legacy else { - return Ok(None); - }; - let _lane = self.key_lock(address).await; - let Some(content) = legacy.lmdb.get(address).await? else { - return Ok(None); - }; - // Only if the file really is gone: a concurrent write or repair may have put a - // good one back while this was waiting for the lock. - if !self.files.is_indexed(address) { - legacy.only.write().insert(*address); - debug!( - "Chunk {} served from the legacy environment and re-queued for copying", - hex::encode(address) - ); + /// Held across await points, unlike the write lanes, so a whole logical transition for + /// a key excludes another. `None` only if the table were empty, which it is not. + async fn key_lock(&self, address: &XorName) -> Option> { + let lane = address.last().copied().unwrap_or(0) as usize; + match self.key_locks.get(lane) { + Some(lock) => Some(lock.lock().await), + None => None, } - Ok(Some(content)) } - /// Retrieve raw chunk bytes without content-address verification. - /// - /// # Errors - /// - /// Returns [`Error::Storage`] on an I/O failure. - pub async fn get_raw(&self, address: &XorName) -> Result>> { - // For the reason given on `get`: retirement must not run in the gap between the - // file going missing and this key being put back in the union view. - let _reading = self.retirement.read().await; - let fallback = self.legacy(); - let from_files = self.files.get_raw(address).await; - match from_files { - Ok(Some(content)) => return Ok(Some(content)), - Ok(None) => {} - // Same rule as `get`: while the legacy environment is there it may have the - // bytes, and this is the read that drives digest audits, possession checks - // and pruning. Answering "no digest" for a chunk the node can still produce - // is a failed audit for nothing. - Err(e) => { - let Some(legacy) = fallback else { - return Err(e); - }; - let _lane = self.key_lock(address).await; - return match legacy.lmdb.get_raw(address).await { - Ok(Some(content)) => Ok(Some(content)), - // Nothing anywhere: report the original failure, not a plain miss. - Ok(None) | Err(_) => Err(e), - }; - } - } - // Deliberately not gated on the legacy-only set. A chunk that was copied and then - // lost its file is not in that set, and the legacy environment is exactly where - // its bytes still are. An LMDB miss is cheap. Goes through the same path as - // `get`, so the key is restored to the union view rather than being served once - // and then quietly retired away. - let Some(legacy) = fallback else { - return Ok(None); - }; - let _lane = self.key_lock(address).await; - let raw = legacy.lmdb.get_raw(address).await?; - let missing_locally = !self.files.is_indexed(address); - if raw.is_some() && missing_locally { - legacy.only.write().insert(*address); - } - Ok(raw) - } - - /// Check whether a chunk is stored, in either backing. - /// - /// An in-memory lookup: no syscall, no I/O, in both phases. - /// - /// # Errors + /// The address content hashes to. /// - /// Never fails. The signature is kept because callers treat an error as "absent". - pub fn exists(&self, address: &XorName) -> Result { - // Legacy first, deliberately. The copier writes the file and only then drops the - // key from the legacy-only set, so a reader that checked files first could - // observe the gap between those two steps and report a chunk the node definitely - // holds as absent. In this order the same interleaving yields a harmless - // duplicate instead. - if self - .legacy() - .is_some_and(|l| l.only.read().contains(address)) - { - return Ok(true); - } - self.files.exists(address) + /// A convenience the old facade offered and callers still use, so it stays with the + /// store rather than making every one of them reach for the client module. + #[must_use] + pub fn compute_address(content: &[u8]) -> XorName { + crate::client::compute_address(content) } - /// Does this node already hold `address` with exactly these bytes, and if it holds a - /// damaged copy, replace it with these? - /// - /// The question a responder has to answer before turning away an offered copy. Plain - /// [`Self::exists`] answers from names alone, and a name can outlive the bytes under - /// it: off Unix a chunk is created under its final name before it is written, so a - /// crash leaves a short file that `exists` reports as a chunk, and bit rot leaves a - /// full-length one. Acknowledging a client on the strength of either throws away the - /// copy that would repair it, and nothing offers it again. + /// Does this store already hold exactly these bytes under this address? /// - /// So this reads. It is affordable because the only caller is the client-facing PUT - /// path, reached when a client offers a chunk this node already has, and because the - /// alternative is keeping a chunk this node cannot serve and being penalised for it at - /// the next audit. + /// Asked by the protocol handler before it accepts a client's PUT, so the answer has + /// to be about the bytes and not about the name. A name on disk is not evidence: the + /// startup scan indexes by name without reading anything, and a partial file can wear a + /// real chunk name. Answering yes on a name would acknowledge a chunk that was never + /// stored and then discard the good copy that had just arrived to replace it. /// - /// `content` must already hash to `address`; the caller checks that before this is - /// reached, and a repair from bytes that do not would be worse than the damage. - /// - /// # Errors - /// - /// Never fails. An unreadable chunk answers `false`, so the offered copy is stored - /// through the ordinary path rather than refused. + /// A chunk that is on disk with the wrong bytes is repaired from what the caller + /// offered rather than turned away, because the caller has already checked those bytes + /// against the address. A chunk that cannot be read this time is not claimed as held + /// and not replaced either: the offer goes through the ordinary write path instead, + /// which never truncates a healthy file on the strength of an unanswered question. pub async fn holds_verified(&self, address: &XorName, content: &[u8]) -> bool { - // Held for the whole check, like every other operation that can reach the legacy - // environment. - let _using_legacy = self.retirement.read().await; - // And the key's own critical section, for the whole of it. Without it the pruner - // can delete both backings between the read and the answer, and the offered copy - // would be turned away for a chunk the node no longer has at all. + // The key's critical section for the whole check, so the answer is a linearizable + // statement about the store: at the moment this returns, the chunk was held and its + // bytes were these. + // + // It does not follow the answer out to the caller. The handler turns a `true` into + // an `AlreadyExists` and sends it afterwards, outside this lock, so a prune landing + // in between still means a peer is told to drop a copy of a chunk this node no + // longer has. Closing that would mean holding a per-key lock across a network + // response, which trades a narrow window for a much worse one. Replication finds + // the key missing and re-offers it. let _lane = self.key_lock(address).await; - if let Some(legacy) = self.legacy() { - if legacy.only.read().contains(address) { - // Held only in the legacy environment. Not taken on trust either: the - // bytes in there can be wrong too, and the copier drops such a key from - // the union when it finds out, which would leave no copy anywhere if this - // had turned the good one away. - if matches!(legacy.lmdb.get_raw(address).await, Ok(Some(bytes)) if bytes == content) - { - return true; - } - warn!( - "Chunk {} is in the legacy environment but its bytes are wrong; \ - storing the copy just offered instead", - hex::encode(address) - ); - if self.files.put(address, content).await.is_err() { - return false; - } - legacy.only.write().remove(address); - return true; - } - } - if !self.files.is_indexed(address) { + if !self.is_indexed(address) { return false; } // No cheap length pre-check. `metadata` failing is not the same as a length that // does not match, and off Unix replacing a chunk truncates it in place, so acting // on an unanswered question would empty a healthy sole copy. The read below // distinguishes them. - match self.files.get_raw(address).await { - // Byte-for-byte what the caller has, and the caller checked those bytes - // against the address before getting here. Nothing is wrong with this file. + match self.get_raw(address).await { + // Byte-for-byte what the caller has, and the caller checked those bytes against + // the address before getting here. Nothing is wrong with this file. Ok(Some(stored)) if stored == content => { - self.files.note_bytes_proven_good(address); + self.note_bytes_proven_good(address); true } Ok(_) => { @@ -841,151 +1114,500 @@ impl ChunkStore { copy just offered", hex::encode(address) ); - // Recorded before the repair is attempted, not after it succeeds. A - // repair can fail for capacity or I/O, and a chunk proven wrong that goes - // on looking healthy leaves a cached pre-retirement pass covering it, - // which deletes the legacy copy the repair would have come from. - self.files.note_known_wrong(address); - self.files.repair(address, content).await.is_ok() + // Recorded before the repair is attempted, not after it succeeds. A repair + // can fail for capacity or I/O, and a chunk proven wrong that goes on + // looking healthy is one the node keeps answering for. + self.note_known_wrong(address); + self.repair_holding_the_lane(address, content).await.is_ok() } // Unanswerable this time. Not claimed as held, so the offer goes through the // ordinary path, which writes it rather than replacing anything. + Err(_) => false, + } + } + + /// Decide what to do about a write of a chunk the index already names. + /// + /// `None` means the index was wrong and there is nothing on disk, so the caller + /// publishes it as new. Everything else is the answer. + async fn settle_indexed_duplicate( + &self, + address: &XorName, + content: &[u8], + ) -> Option> { + match self.stored_bytes_match(address).await { + StoredBytes::Good => { + trace!("Chunk {} already exists", hex::encode(address)); + { + let mut stats = self.stats.write(); + stats.duplicates = stats.duplicates.saturating_add(1); + } + Some(Ok(false)) + } + StoredBytes::Wrong => { + warn!( + "Chunk {} is indexed but its bytes are wrong; replacing it with the \ + copy just offered", + hex::encode(address) + ); + Some( + self.repair_holding_the_lane(address, content) + .await + .map(|()| true), + ) + } + // Indexed but gone: publish it fresh rather than replacing something that is + // not there. + StoredBytes::Absent => None, + // Unanswerable this time. Do not touch what is there, and do not tell the + // caller the chunk is safely stored either: a client would take that as an + // acknowledgement and drop the only other copy. The index entry stays, for + // the reason given on the same case after publication. + StoredBytes::Unreadable => Some(Err(Error::Storage(format!( + "Chunk {} is indexed but could not be read to check it. Not replacing it, \ + and not reporting it as stored.", + hex::encode(address) + )))), + } + } + + /// Whether the file already stored under `address` really hashes to it. + async fn stored_bytes_match(&self, address: &XorName) -> StoredBytes { + match self.get_raw(address).await { + Ok(Some(bytes)) if crate::client::compute_address(&bytes) == *address => { + // A read that hashed. It settles both questions. + self.clear_suspect(address); + self.clear_known_wrong(address); + StoredBytes::Good + } + Ok(Some(_)) => { + self.mark_known_wrong(address); + StoredBytes::Wrong + } + Ok(None) => StoredBytes::Absent, + // NOT the same as wrong. A file that could not be read this once may be + // perfectly good, and off Unix replacing it means opening it with `truncate`, + // which would destroy a healthy sole copy on the strength of a transient + // fault. Say so and let the caller leave it alone. Err(e) => { - warn!("Could not read {} to check it: {e}", hex::encode(address)); - false + debug!("Could not read {} to check it: {e}", hex::encode(address)); + self.mark_suspect(address); + StoredBytes::Unreadable + } + } + } + + /// Replace the file behind an address with bytes that hash to it. + /// + /// Unlike [`Self::put`], this deliberately publishes **over** an existing name, for + /// repairing a file whose bytes no longer hash to their own. Doing it as + /// delete-then-put would leave a window with no copy at all. + /// + /// **Only call this once a read has shown the existing bytes are wrong.** On Unix the + /// replacement is atomic and a failure leaves the old file untouched. Off Unix it is + /// not: there is no durable rename there, so the existing file is truncated and + /// rewritten in place, and a crash part-way leaves a mixture. That is tolerable when + /// the bytes being replaced were already known to be wrong, and is data loss when they + /// were not. Nothing enforces the precondition, which is why it is stated here. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] if `content` does not hash to `address`, or the write + /// fails. + pub async fn repair(&self, address: &XorName, content: &[u8]) -> Result<()> { + let _lane = self.key_lock(address).await; + self.repair_holding_the_lane(address, content).await + } + + /// The body of [`Self::repair`], for callers that already hold the key's lane. + /// + /// Split because the lane is a `tokio::sync::Mutex` and so is not reentrant: every + /// internal caller reaches this while holding it, and taking it again would deadlock + /// the task on itself. + async fn repair_holding_the_lane(&self, address: &XorName, content: &[u8]) -> Result<()> { + // The same ceiling `put` enforces. Without it a repair can install bytes the read + // path will refuse for ever, which is a chunk that verifies as present and can + // never be served. + if content.len() > MAX_CHUNK_SIZE { + return Err(Error::Storage(format!( + "Refusing to repair {} with {} bytes, over the {MAX_CHUNK_SIZE} byte \ + maximum", + hex::encode(address), + content.len() + ))); + } + let computed = crate::client::compute_address(content); + if computed != *address { + return Err(Error::Storage(format!( + "Refusing to repair {} with content that hashes to {}", + hex::encode(address), + hex::encode(computed) + ))); + } + // The replacement exists alongside the original until the rename, so the room for + // it has to be there first. Reserved rather than merely checked: a plain check + // passes against a cached measurement, so concurrent repairs and PUTs can each be + // admitted against the same headroom and cross the reserve together. + // Moved into the work below, so it is released when the write finishes rather + // than when its caller stops waiting. A caller that goes away otherwise frees + // room that the detached write is still about to consume. + let reservation = self.capacity.reserve(content.len() as u64)?; + + let shard = self.chunks_dir.join(shard_name(address)); + let final_path = shard.join(hex::encode(address)); + let temp_path = shard.join(self.next_temp_name()); + let payload = content.to_vec(); + let lanes = Arc::clone(&self.write_lanes); + let index = Arc::clone(&self.index); + let shards_present = Arc::clone(&self.shards_present); + let chunks_dir = self.chunks_dir.clone(); + let lane = shard_index(address); + let key = *address; + let in_flight = self.begin_write(address); + let lease = Arc::clone(&self.lock); + let capacity = Arc::clone(&self.capacity); + let suspect = Arc::clone(&self.suspect); + let known_wrong = Arc::clone(&self.known_wrong); + + self.blocking_tracker + .spawn_blocking(move || -> Result<()> { + let _in_flight = in_flight; + let _lease = lease; + let _reservation = reservation; + let _lane = lanes.get(lane).map(parking_lot::Mutex::lock); + ensure_shard_dir(&chunks_dir, &shard, lane, &shards_present)?; + write_and_replace(&temp_path, &final_path, &payload, &shard)?; + index.write().insert(key); + // Settled here rather than after the await. The replacement has landed + // and hashes to its own name, so nothing is wrong with this chunk any + // more; a caller that stopped waiting would otherwise leave a healthy + // file excluded from everything the node claims to hold, and the + // measurement believing the store is a chunk smaller than it is. + suspect.write().remove(&key); + known_wrong.write().remove(&key); + capacity.invalidate(); + Ok(()) + }) + .await + .map_err(|e| Error::Storage(format!("Chunk store repair task failed: {e}")))??; + + // Everything the success means was recorded by the work that succeeded: the + // reservation released, the marks cleared, the measurement thrown away. Released + // rather than committed because a repair is not a new chunk, and the measurement + // discarded rather than adjusted because the file it replaced may have been + // shorter, which is exactly the case a repair fixes. + debug!("Repaired chunk {}", hex::encode(address)); + Ok(()) + } + + /// Retrieve a chunk, verifying it against its address when configured to. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] on an I/O failure, or when verification fails. A + /// chunk whose bytes do not hash to its name is removed and dropped from the index + /// before the error is returned, so it leaves `all_keys()` and ordinary replication + /// repairs it. + pub async fn get(&self, address: &XorName) -> Result>> { + let Some(content) = self.read_file(address).await? else { + trace!("Chunk {} not found", hex::encode(address)); + return Ok(None); + }; + + if self.config.verify_on_read { + let computed = crate::client::compute_address(&content); + if computed != *address { + { + let mut stats = self.stats.write(); + stats.verification_failures = stats.verification_failures.saturating_add(1); + } + warn!( + "Chunk verification failed: expected {}, computed {}", + hex::encode(address), + hex::encode(computed) + ); + // Said before it is acted on. Removing the file can fail or be cancelled, + // and a chunk proven wrong that goes on looking healthy is one the node + // keeps committing to and keeps being audited for. + self.mark_known_wrong(address); + self.quarantine_corrupt(address).await; + return Err(Error::Storage(format!( + "Chunk verification failed for {}", + hex::encode(address) + ))); } } + + if self.config.verify_on_read { + // The bytes hashed to their name. Whatever this store thought was wrong with + // them is not wrong with them, and a mark that outlives the fault it + // describes means the node can serve a chunk it will not claim, commit or + // offer. + self.clear_known_wrong(address); + } + + let len = content.len() as u64; + { + let mut stats = self.stats.write(); + stats.chunks_retrieved = stats.chunks_retrieved.saturating_add(1); + stats.bytes_retrieved = stats.bytes_retrieved.saturating_add(len); + } + debug!("Retrieved chunk {} ({len} bytes)", hex::encode(address)); + Ok(Some(content)) + } + + /// Retrieve raw chunk bytes without content-address verification. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] on an I/O failure. + pub async fn get_raw(&self, address: &XorName) -> Result>> { + self.read_file(address).await + } + + /// Check whether a chunk is stored. + /// + /// An in-memory lookup: no syscall, no I/O. + /// + /// # Errors + /// + /// Never fails. The signature keeps the shape the LMDB store had, because callers + /// treat the error as "assume absent". + pub fn exists(&self, address: &XorName) -> Result { + if self.is_unservable(address) { + return Ok(false); + } + Ok(self.is_indexed(address)) + } + + /// Is this chunk one the node must not answer for? + #[must_use] + fn is_unservable(&self, address: &XorName) -> bool { + self.suspect.read().contains(address) || self.known_wrong.read().contains(address) + } + + /// Is this chunk in the index, whether or not it can currently be read? + /// + /// The physical question, as against [`Self::exists`]'s question about what the node + /// is willing to claim. The migration must ask this one: a suspect chunk is still a + /// file this store has, and treating it as absent would put the key in the legacy-only + /// set, from where the union view advertises it again — a key the node claims through + /// one view and cannot serve through either. + #[must_use] + pub fn is_indexed(&self, address: &XorName) -> bool { + self.index.read().contains(address) } - /// Delete a chunk from both backings. + /// Delete a chunk, returning whether it was present. /// - /// A logical delete has to reach the legacy environment too, or the union view would - /// resurrect the key on the next read. It frees no space there — only removing the - /// environment whole does that — but it keeps the two views honest. + /// `unlink` returns the blocks to the filesystem immediately. That is the whole + /// point of this store: no free list, no compaction, no free space required to + /// reclaim space. /// /// # Errors /// - /// Returns [`Error::Storage`] if a file exists but cannot be removed. + /// Returns [`Error::Storage`] if the file exists but cannot be removed. The index + /// keeps the key in that case, because the bytes are still on disk. pub async fn delete(&self, address: &XorName) -> Result { - // Shared, like every other operation that touches the legacy environment. Without - // it, retirement takes the exclusive guard and then waits for the environment to - // go idle while deletes keep starting new work in it, and the wait never ends. - let _using_legacy = self.retirement.read().await; + // The key's whole critical section, held across the wait below and the unlink. let _lane = self.key_lock(address).await; // Behind whatever is already writing this key, and only this key. A write's // blocking half outlives the future that started it, so one landing after this - // would put back a chunk the node had decided to prune. - self.files.wait_for_write(address).await; - // Legacy first, and only then the in-memory views. The other order removes the - // key from `only` and then, if the legacy delete fails, leaves bytes that live - // solely in the legacy store and are invisible to `exists`, `all_keys` and the - // pre-retirement verification, so retirement would take the only copy. - let from_legacy = match self.legacy() { - Some(legacy) => { - // A write for this key that nobody waited for may still be queued behind - // this delete. Letting it land afterwards would resurrect the key: the - // next reconciliation finds it in the environment and puts it back on the - // copier's list, undoing a prune the node decided on. Waited out here, - // holding the lane, so the delete is genuinely last. - // - // Both halves. A write has an environment half and a file half, either of - // which can be the one still running, and draining only the first leaves - // the second free to publish the file after this has deleted it. - // - // The environment half is found through the journal, which only dual - // writes keep. The file half is asked of the file store directly, because - // the copier and the repair path also spawn file writes and neither goes - // near that journal: using it as a proxy for "is anything writing this - // key" was a scope assumption, not a fact. - if legacy.pending.read().contains_key(address) { - legacy.lmdb.wait_idle().await; - } - let deleted = legacy.lmdb.delete(address).await?; - let was_only = legacy.only.write().remove(address); - // Every announcement for this key, not one of them: the drain above waited - // out whatever was in flight and this delete is deliberately last. - legacy.pending.write().remove(address); - deleted || was_only - } - None => false, - }; - let from_files = self.files.delete(address).await?; - Ok(from_files || from_legacy) + // would put back a chunk the node had decided to prune, and the next thing to look + // would find it in a store that no longer claims it. + self.wait_for_write(address).await; + + let path = self.chunk_path(address); + let lanes = Arc::clone(&self.write_lanes); + let index = Arc::clone(&self.index); + let lane = shard_index(address); + let key = *address; + // Carried into the closure for the reason `put`, `repair` and the startup scan + // carry it: this work outlives the future that started it, so a cancelled caller + // that drops the last `ChunkStore` would otherwise release the directory to another + // process while an unlink is still queued against it. Deleting is the operation + // where that matters most. + let lease = Arc::clone(&self.lock); + + let (existed, freed) = self + .blocking_tracker + .spawn_blocking(move || -> Result<(bool, u64)> { + let _lease = lease; + let _lane = lanes.get(lane).map(parking_lot::Mutex::lock); + let len = std::fs::metadata(&path).map_or(0, |m| m.len()); + let removed = match std::fs::remove_file(&path) { + Ok(()) => { + // Without this a crash can resurrect the entry on ext4, XFS, + // btrfs and APFS: the unlink is in the page cache, the directory + // is not. + if let Some(shard) = path.parent() { + fsync_dir_best_effort(shard); + } + true + } + // Already gone: the index was stale. Still a successful delete as + // far as the caller is concerned. + Err(e) if e.kind() == ErrorKind::NotFound => false, + Err(e) => { + return Err(Error::Storage(format!( + "Failed to delete chunk file {}: {e}", + path.display() + ))) + } + }; + // Index only after the filesystem operation has succeeded. On the error + // path above the entry stays, because the bytes are still on disk. + let was_indexed = index.write().remove(&key); + Ok((removed || was_indexed, if removed { len } else { 0 })) + }) + .await + .map_err(|e| Error::Storage(format!("Chunk store delete task failed: {e}")))??; + + if freed > 0 { + self.capacity.record_removed(freed); + debug!("Deleted chunk {}", hex::encode(address)); + } + Ok(existed) } - /// Every stored key, in ascending order, across both backings. + /// Return every stored key, in ascending order. /// - /// The order is a correctness requirement: the commitment builder truncates the - /// responsible subset with `take(cap)` *before* the Merkle tree sorts it, so an - /// unstable order would make the node's published commitment depend on iteration - /// luck rather than on what it holds. + /// The order is a correctness requirement, not a convenience: the commitment + /// builder truncates the responsible subset with `take(cap)` before the Merkle tree + /// sorts it, so an unstable order would make the node's published commitment depend + /// on iteration luck. /// /// # Errors /// - /// Returns [`Error::Storage`] if the file index cannot be read. + /// Never fails. The signature matches the LMDB store's. + // Async without awaiting anything, deliberately: the whole point of this store is + // that the key set is already in memory. Callers are spread across the replication + // engine and cannot all be de-async'd in this change. + // + // Two lint names because they were renamed between toolchains, and `unknown_lints` + // so whichever one the compiler in use has never heard of stays quiet. + #[allow(unknown_lints)] + #[allow(clippy::unused_async, clippy::unused_async_trait_impl)] pub async fn all_keys(&self) -> Result> { - // Legacy first, for the reason given on `exists`. `merge_sorted` drops the - // duplicate that the overlap produces. - let legacy_only: Vec = self - .legacy() - .map(|l| l.only.read().iter().copied().collect()) - .unwrap_or_default(); - let file_keys = self.files.all_keys().await?; - if legacy_only.is_empty() { - return Ok(file_keys); + // Copied out first so neither lock is held while the other is taken, and so the + // usual case, where nothing is suspect, costs one clone of an empty set. + let mut unservable: HashSet = self.suspect.read().clone(); + unservable.extend(self.known_wrong.read().iter().copied()); + let keys = self.index.read().clone(); + if unservable.is_empty() { + return Ok(keys.into_iter().collect()); } - Ok(merge_sorted(&file_keys, legacy_only.iter())) + Ok(keys + .into_iter() + .filter(|key| !unservable.contains(key)) + .collect()) } - /// The keys the commitment builder should commit to. + /// Stop answering for a chunk this store could not read. /// - /// While the node is still bridging this is the whole union, because it can still - /// serve all of it and dropping the claim early would collapse its commitment (and - /// with it its quoted price) for no reason. Once it has settled on what it will keep, - /// this narrows to the file-backed set, which is exactly the point at which the node - /// stops claiming keys it is about to give up. + /// The file stays. It may be perfectly good and unreadable only for the moment, and + /// deleting it, or dropping it from the index, is how a chunk ends up in neither this + /// store's view nor the legacy one, which is what retirement destroys. /// - /// # Errors + /// What does change is what the node says about it. A chunk it cannot read is one it + /// cannot serve, and claiming it anyway puts the key in signed commitments, answers + /// presence probes with a yes, suppresses the replication that would repair it, and + /// earns a penalty at the next commitment-bound audit. Those penalties are not + /// suspended. + fn mark_suspect(&self, address: &XorName) { + if self.suspect.write().insert(*address) { + warn!( + "Chunk {} is on disk but could not be read; this node stops answering for \ + it until a read succeeds", + hex::encode(address) + ); + } + } + + /// Stop answering for a chunk a read has proven wrong. /// - /// Returns [`Error::Storage`] if the file index cannot be read. - pub async fn committable_keys(&self) -> Result> { - match self.migration_phase() { - MigrationPhase::Bridging => self.all_keys().await, - MigrationPhase::Committed | MigrationPhase::FilesOnly => self.files.all_keys().await, + /// Unlike a chunk that merely could not be read, a later read does not clear this. + /// The bytes are wrong, and reading them again says the same thing; only replacing + /// them or removing them settles it. + /// + /// For callers outside this module that have proven it themselves. + pub fn note_known_wrong(&self, address: &XorName) { + self.mark_known_wrong(address); + } + + /// Stop answering for a chunk a read has proven wrong. + fn mark_known_wrong(&self, address: &XorName) { + if self.known_wrong.write().insert(*address) { + warn!( + "Chunk {} does not match its name; this node stops answering for it until \ + it is repaired or removed", + hex::encode(address) + ); + } + } + + /// A caller outside this module has proven the stored bytes are right. + pub fn note_bytes_proven_good(&self, address: &XorName) { + self.clear_known_wrong(address); + self.clear_suspect(address); + } + + /// Answer for a chunk again, after it has been replaced or removed. + fn clear_known_wrong(&self, address: &XorName) { + self.known_wrong.write().remove(address); + } + + /// Answer for a chunk again, after a read that worked. + fn clear_suspect(&self, address: &XorName) { + if !self.suspect.read().contains(address) { + return; + } + if self.suspect.write().remove(address) { + info!( + "Chunk {} could be read again; this node answers for it once more", + hex::encode(address) + ); } } - /// Number of chunks currently stored, counted across both backings without - /// double-counting a chunk that is in each. + /// Number of chunks currently stored. + /// + /// The physical count: every name in the index, including chunks the node has stopped + /// answering for because a read found them wrong or could not read them at all. It is + /// deliberately not the same number as `all_keys().len()`, which is what the node is + /// willing to claim and so leaves those out. + /// + /// Anything asking "how much is on this disk" wants this one, and that is what its + /// callers ask: the migration's progress, the storage stats, and the size an audit is + /// built for. Anything asking "what will this node answer for" wants `all_keys`. + /// Quietly filtering this one would move all three of those without saying so, which + /// is why the difference is written down here rather than removed. /// /// # Errors /// - /// Never fails. + /// Never fails. The signature matches the LMDB store's. pub fn current_chunks(&self) -> Result { - let (files, legacy) = self.split_counts(); - Ok(files.saturating_add(legacy)) + Ok(self.index.read().len() as u64) } - /// Operation statistics. - /// - /// The cumulative counters are the file store's; `current_chunks` is the union. + /// Operation statistics, with the live chunk count filled in. #[must_use] pub fn stats(&self) -> StorageStats { - let mut stats = self.files.stats(); - stats.current_chunks = self.current_chunks().unwrap_or(0); + let mut stats = self.stats.read().clone(); + stats.current_chunks = self.index.read().len() as u64; stats } - /// Compute a content address (BLAKE3 hash). + /// The node root directory this store was configured with. #[must_use] - pub fn compute_address(content: &[u8]) -> XorName { - crate::client::compute_address(content) + pub fn root_dir(&self) -> &Path { + &self.config.root_dir } - /// The node root directory. + /// The directory holding the shard tree. #[must_use] - pub fn root_dir(&self) -> &Path { - &self.config.root_dir + pub fn chunks_dir(&self) -> &Path { + &self.chunks_dir } /// Reject work early when the disk cannot take another chunk at all. @@ -994,46 +1616,48 @@ impl ChunkStore { /// /// Returns [`Error::Storage`] when free space is below the configured reserve. pub fn check_capacity(&self) -> Result<()> { - self.files.check_capacity() + self.capacity.check(0) } - /// Whether the store can take a write at all right now. - /// - /// Answered by the file store, which is where writes land. The legacy environment's - /// own verdict is deliberately not consulted: it accounts for pages it can reuse - /// internally, and a reusable page in a store this node is moving *off* says nothing - /// about whether the file it is about to write will fit. + /// Three-way answer to "can this store take a write right now". /// - /// Three-way, not two. The verification cycle treats `Full` as a standing condition - /// worth minutes of backoff, so folding a failed free-space query into it would latch - /// a transient filesystem hiccup into a stall on a node that is not full at all. + /// Kept distinct from [`Self::check_capacity`] because a failed free-space query and a + /// genuinely full disk are not the same thing, and the replication verification cycle + /// depends on the difference: a full disk is a standing condition worth minutes of + /// backoff, while a `statvfs` that failed says nothing about available space and may + /// well succeed on the next pass. #[must_use] - pub(crate) fn capacity_verdict(&self) -> crate::storage::CapacityVerdict { - self.files.capacity_verdict() + pub fn capacity_verdict(&self) -> crate::storage::CapacityVerdict { + match self.capacity.measure_available() { + Some(available) if available < self.capacity.reserve => { + crate::storage::CapacityVerdict::Full + } + Some(_) => crate::storage::CapacityVerdict::Writable, + None => crate::storage::CapacityVerdict::Unknown, + } } /// Reject work early when the disk cannot take `bytes` more. /// - /// Free bytes alone stopped being a sufficient answer once chunks became files. - /// /// # Errors /// /// Returns [`Error::Storage`] when the write would not fit above the reserve. pub fn check_capacity_for(&self, bytes: u64) -> Result<()> { - self.files.check_capacity_for(bytes) + self.capacity.check(bytes) } - /// Charge `bytes` against the disk before writing them. + /// Charge `bytes` against the disk before writing them, releasing the charge if the + /// write does not happen. /// /// For the pointer store, which keeps its own files on this disk. Checking capacity - /// and then writing is a race the file store names explicitly: concurrent writers all - /// pass one cached measurement and cross the reserve together. + /// and then writing is a race [`CapacityGuard::reserve`] exists to close: concurrent + /// writers all pass one cached measurement and cross the reserve together. /// /// # Errors /// /// Returns [`Error::Storage`] if the disk cannot take `bytes` more. - pub(crate) fn reserve(&self, bytes: u64) -> Result { - self.files.reserve_bytes(bytes) + pub(crate) fn reserve(&self, bytes: u64) -> Result { + self.capacity.reserve(bytes) } /// Credit `bytes` deleted from the pointer store back to the disk budget. @@ -1041,4024 +1665,2443 @@ impl ChunkStore { /// The counterpart of [`Self::reserve`]: a record the pointer store removes /// gives its charge back, as a deleted chunk does. pub(crate) fn release(&self, bytes: u64) { - self.files.release_bytes(bytes); + self.capacity.record_removed(bytes); } - /// `(written_since, in_flight)` from the capacity guard. Tests only. + /// `(written_since, in_flight)` from the capacity guard. Tests only: it is + /// how a released charge is told apart from a stranded one. #[cfg(test)] pub(crate) fn capacity_counters(&self) -> (u64, u64) { - self.files.capacity_counters() + self.capacity.counters() } - /// What a payload of `bytes` costs the disk. Tests only. + /// What a payload of `bytes` actually costs the disk, rounded to the + /// allocation unit. Tests only, so they can assert exact counter deltas + /// rather than "it went up" — which a charge of one allocation unit would + /// satisfy for several payloads at once. #[cfg(test)] pub(crate) fn capacity_charge_for(bytes: u64) -> u64 { - crate::storage::FileStore::capacity_charge_for(bytes) - } - - /// Wait until every blocking task in either backing has finished. - pub async fn wait_idle(&self) { - self.files.wait_idle().await; - if let Some(legacy) = self.legacy() { - legacy.lmdb.wait_idle().await; - } - } - - // ── Migration ─────────────────────────────────────────────────────────── - - /// Where the node is in the migration. - #[must_use] - pub fn migration_phase(&self) -> MigrationPhase { - self.state.read().phase + CapacityGuard::charge(bytes) } - /// A snapshot of the persisted migration marker. + /// Test-only handle to the put gate. + /// + /// Hold the write half to park the next write inside its blocking closure, for + /// example to prove that shutdown waits for a write whose awaiter was dropped. + #[cfg(any(test, feature = "test-utils"))] #[must_use] - pub fn migration_state(&self) -> MigrationState { - self.state.read().clone() + pub fn test_put_gate(&self) -> Arc> { + Arc::clone(&self.test_put_gate) } - /// The migration settings this store was built with. - #[must_use] - pub fn migration_config(&self) -> &MigrationConfig { - &self.config.migration + /// Test-only handle to the gate that parks a put before it registers itself. + #[cfg(test)] + fn test_pre_registration_gate(&self) -> Arc> { + Arc::clone(&self.test_pre_registration_gate) } - /// Whether a legacy environment is still open. - #[must_use] - pub fn has_legacy(&self) -> bool { - self.legacy.read().is_some() + /// Test-only: how many puts have reached the pre-registration gate. + #[cfg(test)] + fn test_reached_pre_registration(&self) -> u64 { + self.test_reached_pre_registration + .load(std::sync::atomic::Ordering::Acquire) } - /// The keys the legacy environment still holds alone, ascending. - #[must_use] - pub fn legacy_only_keys(&self) -> Vec { - self.legacy() - .map(|l| l.only.read().iter().copied().collect()) - .unwrap_or_default() + /// Register a write of `address` and hand back the token that clears it. + /// + /// The token must be moved into the blocking closure that does the work, so the entry + /// is cleared by the thread that finishes rather than by a caller that may be gone. + fn begin_write(&self, address: &XorName) -> WriteInFlight { + *self.writing.lock().entry(*address).or_insert(0) += 1; + WriteInFlight { + writing: Arc::clone(&self.writing), + finished: Arc::clone(&self.write_finished), + address: *address, + } } - /// Bytes the legacy environment occupies, as the filesystem sees it. - #[must_use] - pub fn legacy_bytes(&self) -> u64 { - std::fs::metadata(self.legacy_env_dir.join(LEGACY_DATA_FILE)).map_or(0, |m| m.len()) + /// Wait until nothing is part-way through writing `address`. + /// + /// For callers that must be last: a delete whose key still has a write in flight + /// would be undone by that write landing afterwards. + pub async fn wait_for_write(&self, address: &XorName) { + loop { + // Registered before the check, so a clear between the two is not missed. + let waiting = self.write_finished.notified(); + if !self.writing.lock().contains_key(address) { + return; + } + waiting.await; + } } - /// Test-only handle to the file store's put gate. + /// How many blocking tasks this store currently has in flight. Tests only. + /// + /// Lets a test wait for work to have actually started rather than guessing at a + /// delay, which is the difference between a test that proves something and one that + /// passes because the machine was quick. #[cfg(any(test, feature = "test-utils"))] #[must_use] - pub fn test_put_gate(&self) -> Arc> { - self.files.test_put_gate() + pub(crate) fn tasks_in_flight(&self) -> usize { + self.blocking_tracker.len() } - /// Test-only count of running or queued file-store blocking operations. + /// Test-only count of running or queued blocking operations, for harnesses outside + /// this crate. #[cfg(any(test, feature = "test-utils"))] #[must_use] pub fn test_file_tasks_in_flight(&self) -> usize { - self.files.tasks_in_flight() + self.tasks_in_flight() } - /// Test-only: adjust the persisted migration marker directly. + /// Wait until every blocking task this store spawned has finished. /// - /// Real transitions go through [`Self::commit_to_files`] and - /// [`Self::note_commitment_rebuilt`]; this exists so a test can put a store into a - /// state that would otherwise take hours of wall clock to reach. - #[cfg(any(test, feature = "test-utils"))] - pub fn force_migration_state(&self, f: F) { - f(&mut self.state.write()); - } - - /// Copy up to `keys.len()` chunks out of the legacy environment into files. - /// - /// Stops as soon as free space would fall below `slack` above the configured - /// reserve, so a migration never fills the disk it is trying to free. - /// - /// # Errors - /// - /// Returns [`Error::Storage`] only for failures that are not per-key: a per-key - /// problem is counted in the report and the pass continues. - pub async fn copy_batch( - &self, - keys: &[XorName], - slack: u64, - throttle_mib_per_sec: u64, - shutdown: &CancellationToken, - ) -> Result { - let mut report = CopyReport::default(); - let Some(legacy) = self.legacy() else { - return Ok(report); - }; - - for key in keys { - // Checked per chunk, not per batch. Everything here is idempotent and - // re-derived at the next start, so stopping between two chunks costs nothing - // and stops shutdown waiting out a whole pass. - if shutdown.is_cancelled() { - break; - } - let lane = self.key_lock(key).await; - // Re-checked inside the critical section. A prune that landed while this - // pass was running has already taken the key out of the legacy-only set, and - // copying it now would resurrect a chunk the node deliberately deleted. - if !legacy.only.read().contains(key) { - continue; - } - // Physically, again: a chunk the store holds and cannot read is not one to - // copy over the top of, and it is not legacy-only either. - if self.files.is_indexed(key) { - legacy.only.write().remove(key); - continue; + /// Dropping the awaiting future does not cancel a `spawn_blocking` closure, so + /// shutdown has to wait for the closure itself. + pub async fn wait_idle(&self) { + self.blocking_tracker.close(); + self.blocking_tracker.wait().await; + self.blocking_tracker.reopen(); + } + + /// Absolute path of a chunk file. + fn chunk_path(&self, address: &XorName) -> PathBuf { + self.chunks_dir + .join(shard_name(address)) + .join(hex::encode(address)) + } + + /// A temp name unique to this store instance, and distinguishable from a chunk name. + /// + /// The nonce matters: two `ChunkStore`s on one root in one process share a PID, and a + /// recycled PID collides with an age-gated leftover. Either way `create_new` would + /// fail and surface as a spurious write error. + fn next_temp_name(&self) -> String { + let seq = self.temp_seq.fetch_add(1, Ordering::Relaxed); + format!( + "{TEMP_PREFIX}{}.{:08x}.{seq}", + std::process::id(), + self.nonce + ) + } + + /// Read a chunk file, dropping the index entry if the file has vanished. + async fn read_file(&self, address: &XorName) -> Result>> { + let path = self.chunk_path(address); + let read = self + .blocking_tracker + .spawn_blocking(move || -> Result>> { + match open_regular(&path) { + Ok(Some(f)) => read_bounded(f, &path).map(Some), + Ok(None) => Ok(None), + Err(e) => Err(e), + } + }) + .await + .map_err(|e| Error::Storage(format!("Chunk store read task failed: {e}")))?; + + // Every read decides the question, not only the ones that were checking. A read + // that failed means this chunk cannot be served, whoever asked; a read that + // worked means it can be, whoever asked. Doing this anywhere else leaves a key + // stuck unadvertised after the fault has cleared, or advertised after it has not. + let read = match read { + Ok(read) => { + self.clear_suspect(address); + read } - // Reserve room for a full chunk plus the slack floor before reading, so the - // copier stops with headroom rather than on a failed write. - if self.files.check_capacity_for(slack).is_err() { - report.stopped_for_space = true; - break; + Err(e) => { + self.mark_suspect(address); + return Err(e); } + }; - let Some(bytes) = legacy.lmdb.get_raw(key).await? else { - report.vanished += 1; - legacy.only.write().remove(key); - continue; - }; - let len = bytes.len() as u64; - - match self.files.put(key, &bytes).await { - Ok(_) => { - legacy.only.write().remove(key); - report.copied += 1; - report.bytes += len; + if read.is_none() && self.forget_if_absent(address).await { + // The file went away underneath us. Stop advertising the key so the close + // group notices the shortfall and replication puts it back. + warn!( + "Chunk {} is indexed but its file is missing; dropped from the index so \ + replication can repair it", + hex::encode(address) + ); + } + Ok(read) + } + + /// Drop an index entry whose file is genuinely gone. + /// + /// Re-checks under the address's write lane, so a chunk republished between the + /// failing read and this call keeps its entry. + async fn forget_if_absent(&self, address: &XorName) -> bool { + // Not suspect any more: it is not unreadable, it is not there. + self.clear_suspect(address); + let path = self.chunk_path(address); + let lanes = Arc::clone(&self.write_lanes); + let index = Arc::clone(&self.index); + let lane = shard_index(address); + let key = *address; + // The bump happens inside the closure, with the mutation it describes. The + // closure runs to completion on its own thread whether or not anyone is still + // awaiting it, so bumping after the await is skipped entirely when a shutdown + // drops the caller — and the index change it was meant to announce still lands. + // A cached pre-retirement proof would then stay valid over a store that had + // quietly lost a chunk. + self.blocking_tracker + .spawn_blocking(move || { + let _lane = lanes.get(lane).map(parking_lot::Mutex::lock); + if path.exists() { + return false; } - Err(e) => { - let message = format!("{e}"); - // Bigger than this build will ever serve. The legacy store took it - // through an API with no size bound; the file store will not, and no - // amount of retrying changes that. Counted as unusable and removed, - // like a record whose bytes do not match, or one such record would - // stop this node and every node sharing its disk from ever reclaiming - // space. - if message.contains("byte maximum") { - warn!( - "Chunk {} in the legacy environment is larger than this build \ - will store; removing it. It cannot be served either way.", - hex::encode(key) - ); - match legacy.lmdb.delete(key).await { - Ok(_) => { - legacy.only.write().remove(key); - report.unusable += 1; - } - Err(e) => warn!( - "Oversized chunk {} could not be removed from the legacy \ - environment: {e}. The environment stays.", - hex::encode(key) - ), - } - continue; - } - if message.contains("Content address mismatch") { - // The legacy bytes do not hash to their own key, so this chunk - // cannot be reproduced and was never servable. Stop advertising - // it rather than carrying a key we cannot answer for. - // - // Deleted from the environment too, and only dropped from the key - // set once that has worked. Leaving the record behind puts the - // key in neither view, which the pre-retirement pass reads as a - // chunk to protect and puts straight back — and the next copier - // pass drops it again. One malformed record would keep a node, - // and every node sharing its disk, from ever reclaiming space. - warn!( - "Chunk {} in the legacy environment does not match its address; \ - removing it so replication can repair it", - hex::encode(key) - ); - match legacy.lmdb.delete(key).await { - Ok(_) => { - legacy.only.write().remove(key); - report.unusable += 1; - } - Err(e) => warn!( - "Chunk {} does not match its address and could not be \ - removed from the legacy environment: {e}. It stays on the \ - list and the environment stays.", - hex::encode(key) - ), + let forgotten = index.write().remove(&key); + forgotten + }) + .await + .unwrap_or(false) + } + + /// Remove a chunk whose bytes do not match its name, and stop advertising it. + /// + /// Re-reads and re-verifies under the address's write lane first. A read that failed + /// verification is rare enough that paying for one extra read is worth never + /// discarding a chunk that a concurrent write had already repaired. + async fn quarantine_corrupt(&self, address: &XorName) { + let path = self.chunk_path(address); + let lanes = Arc::clone(&self.write_lanes); + let index = Arc::clone(&self.index); + let lane = shard_index(address); + let key = *address; + // For the reason given on `forget_if_absent`: this closure outlives its awaiter, + // and the change it makes has to be announced by the same thread that makes it. + // And the store-lock lease, for the reason `put`, `repair`, `delete` and the + // startup scan carry it: this closure outlives its awaiter, so without it a + // cancelled verification whose caller dropped the last `ChunkStore` would unlink + // inside a directory a second process had already been handed. + let lease = Arc::clone(&self.lock); + let outcome = + self.blocking_tracker + .spawn_blocking(move || -> std::io::Result { + let _lease = lease; + let _lane = lanes.get(lane).map(parking_lot::Mutex::lock); + // Nothing is thrown away without proof. A re-read that fails says the + // question could not be answered this time, not that the bytes are wrong, + // and a repair may have published a good copy since the read that brought + // us here. Treating either as corruption deletes a chunk this node has. + let buf = match open_regular(&path) { + Ok(Some(f)) => read_bounded(f, &path) + .map_err(|e| std::io::Error::other(e.to_string()))?, + Ok(None) => { + index.write().remove(&key); + return Ok(true); } - continue; + Err(e) => return Err(std::io::Error::other(e.to_string())), + }; + if crate::client::compute_address(&buf) == key { + // Repaired between the failing read and now. Leave it alone. + return Ok(false); } - if message.contains("Insufficient disk space") { - report.stopped_for_space = true; - break; + std::fs::remove_file(&path)?; + // The same flush the ordinary delete does, for the same reason: an + // unlink that has not reached the directory can be undone by a power + // loss, and here the entry that comes back is one this node has proven + // wrong. The startup scan would re-index it by name, and the + // known-wrong mark that would otherwise hold it back lives only in + // memory and does not survive the restart, so the node would go back to + // claiming and committing to a chunk it already knows is bad. + if let Some(shard) = path.parent() { + fsync_dir_best_effort(shard); } - return Err(e); - } + index.write().remove(&key); + Ok(true) + }) + .await; + match outcome { + Ok(Ok(true)) => { + self.clear_known_wrong(address); + self.clear_suspect(address); + warn!( + "Removed corrupt chunk file {}; replication will repair it", + hex::encode(address) + ); } - - // Outside the critical section on purpose: at 32 MiB/s a 4 MiB chunk sleeps - // for over a tenth of a second, and a shard lane held for that would stall - // every write sharing its last address byte for the whole pass. - drop(lane); - if let Some(delay) = throttle_delay(len, throttle_mib_per_sec) { - tokio::time::sleep(delay).await; + Ok(Ok(false)) => { + // The re-read hashed and matched: a repair landed between the failing + // read and this one. + self.clear_known_wrong(address); + self.clear_suspect(address); + debug!( + "Chunk {} verified on re-read; leaving it in place", + hex::encode(address) + ); + } + // Still indexed, so it must not still be claimed: the read that brought us + // here proved the bytes wrong, and the node would otherwise go on committing + // to a chunk it knows it cannot serve. + Ok(Err(e)) => { + self.mark_suspect(address); + warn!( + "Corrupt chunk {} could not be removed: {e}. It stays on disk, and \ + this node stops answering for it.", + hex::encode(address) + ); + } + Err(e) => { + self.mark_suspect(address); + warn!("Corrupt-chunk removal task failed: {e}"); } } - Ok(report) } +} - /// Settle on the file-backed set: from now on the node commits only to what it will - /// keep, while still serving everything it ever committed to. - /// - /// # Errors - /// - /// Returns [`Error::Storage`] if the marker cannot be persisted. - pub fn commit_to_files(&self) -> Result<()> { - let shed = self - .legacy() - .map_or(0, |l| l.only.read().len().try_into().unwrap_or(u64::MAX)); - let kept = self.files.current_chunks().unwrap_or(0); - // Written to disk before it is published in memory. The other order leaves this - // process acting as `Committed` (and so committing only to file-backed keys) - // while the marker still says `Bridging`, so a restart would silently undo it. - let candidate = { - let state = self.state.read(); - if state.phase != MigrationPhase::Bridging { - return Ok(()); - } - MigrationState { - phase: MigrationPhase::Committed, - committed_at_unix: None, - rebuilds_since_commit: 0, - shed_key_count: shed, - kept_key_count: kept, - ..state.clone() - } +// ──────────────────────────────────────────────────────────────────────────── +// Free functions +// ──────────────────────────────────────────────────────────────────────────── + +/// Create the destination shard directory if this store has not seen it yet. +/// +/// A newly created directory entry is only durable once its parent is flushed; without +/// that a crash could take the directory and the chunk inside it together. +fn ensure_shard_dir( + chunks_dir: &Path, + dir: &Path, + shard: usize, + present: &parking_lot::Mutex<[bool; SHARD_COUNT]>, +) -> Result<()> { + if present.lock().get(shard).copied().unwrap_or(false) { + return Ok(()); + } + std::fs::create_dir_all(dir).map_err(|e| { + Error::Storage(format!( + "Failed to create shard directory {}: {e}", + dir.display() + )) + })?; + // Load-bearing, like the flush that publishes a chunk into this directory. Until the + // parent is flushed the shard's own entry can be lost, and losing it loses every chunk + // inside it. Reporting the shard present anyway would let the very first chunk written + // into it count as durably stored. + fsync_dir(chunks_dir).map_err(|e| { + Error::Storage(format!( + "Created shard directory {} but could not flush {}: {e}. Not marking the shard \ + usable, because a directory that is not durable cannot hold a chunk that is.", + dir.display(), + chunks_dir.display() + )) + })?; + if let Some(slot) = present.lock().get_mut(shard) { + *slot = true; + } + Ok(()) +} + +/// Shard directory index for an address: its last byte. +fn shard_index(address: &XorName) -> usize { + address.last().copied().unwrap_or(0) as usize +} + +/// Shard directory name for an address: the last two characters of its hex form. +fn shard_name(address: &XorName) -> String { + format!("{:02x}", shard_index(address)) +} + +/// True for a string of hex digits in either case. +fn is_hex_any_case(s: &str) -> bool { + !s.is_empty() && s.bytes().all(|b| b.is_ascii_hexdigit()) +} + +/// Move an entry aside under a name that can never be read as a chunk. +fn quarantine_entry(path: &Path) { + let aside = path.with_extension("not-a-chunk"); + match std::fs::rename(path, &aside) { + Ok(()) => warn!( + "Chunk store: moved {} aside to {}; a name that differs from a chunk name only \ + by case collides with it on Windows and macOS", + path.display(), + aside.display() + ), + Err(e) => warn!( + "Chunk store: {} collides with a chunk name by case folding and could not be \ + moved aside: {e}. Rename or delete it.", + path.display() + ), + } +} + +/// True for a string of lowercase hex digits only. +fn is_lower_hex(s: &str) -> bool { + !s.is_empty() && s.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')) +} + +/// Decode a filename back into the address it names, or `None` if it is not one. +/// +/// Rejects uppercase deliberately. On a case-folding filesystem (NTFS, default APFS) +/// accepting both cases would let one file answer to two index entries. +fn decode_chunk_name(name: &str) -> Option { + if name.len() != CHUNK_NAME_LEN || !is_lower_hex(name) { + return None; + } + let bytes = hex::decode(name).ok()?; + XorName::try_from(bytes.as_slice()).ok() +} + +/// Flush a directory so a rename or creation inside it survives power loss. +/// +/// Best effort by design. Linux and XFS require it, macOS accepts it with undocumented +/// effect, and Windows offers no way to do it at all through the standard library. The +/// content is content-addressed and re-replicable, so a lost directory entry costs a +/// refetch rather than data. Pretending otherwise in the code would be dishonest. +#[cfg(unix)] +fn fsync_dir_best_effort(path: &Path) { + if let Err(e) = fsync_dir(path) { + debug!("Directory flush of {} failed: {e}", path.display()); + } +} + +/// Flush a directory, reporting whether it worked. +/// +/// Used where the answer is load-bearing: a chunk copied out of the legacy store is only +/// durable once its directory entry is, and that copy is what permits the legacy store to +/// be deleted. +#[cfg(unix)] +fn fsync_dir(path: &Path) -> std::io::Result<()> { + File::open(path)?.sync_all() +} + +/// Off Unix there is no way to flush a directory through the standard library, so this +/// reports success without being able to promise anything. +/// +/// That is why the publish path off Unix does not use a rename at all: it creates the +/// chunk under its final name and flushes the file, which Microsoft documents as flushing +/// the creation metadata with it. Directory creation has no equivalent, so the guarantee +/// there rests on the pre-retirement pass, which re-reads every chunk before the legacy +/// store is deleted, and on the operator gate that keeps retirement off a platform until +/// forced power loss has been shown to hold old-or-new on it. +/// +/// Returns a `Result` so the callers that must handle a flush failure on Unix read the +/// same on every platform. +#[cfg(not(unix))] +#[allow(clippy::unnecessary_wraps)] +fn fsync_dir(_path: &Path) -> std::io::Result<()> { + Ok(()) +} + +/// No-op on platforms with no way to flush a directory handle. +#[cfg(not(unix))] +fn fsync_dir_best_effort(_path: &Path) {} + +/// Warn if the deepest chunk path this store can produce is close to `MAX_PATH`. +#[cfg(windows)] +fn check_path_budget(chunks_dir: &Path) { + // Measured absolute, because that is what the filesystem sees. A relative root is the + // case that still fails hard at MAX_PATH, since the standard library's long-path + // handling only applies to paths it resolves as absolute. + let absolute = if chunks_dir.is_absolute() { + chunks_dir.to_path_buf() + } else { + std::env::current_dir() + .map_or_else(|_| chunks_dir.to_path_buf(), |cwd| cwd.join(chunks_dir)) + }; + // `{chunks_dir}\{xy}\{64 hex}` — two separators, two shard characters, 64 name + // characters. + let deepest = absolute.as_os_str().len() + 1 + 2 + 1 + CHUNK_NAME_LEN; + if deepest > WINDOWS_PATH_WARN_LEN { + warn!( + "Chunk file paths will be {deepest} characters, close to the {} character \ + Windows limit. Move the node root closer to the drive letter if writes start \ + failing.", + WINDOWS_PATH_WARN_LEN + ); + } +} + +/// No-op where path length is not a practical constraint. +#[cfg(not(windows))] +fn check_path_budget(_chunks_dir: &Path) {} + +/// Write `bytes` to `path` so a reader sees either the old content or the new. +/// Is this the exact name [`write_file_atomic`] gives its temporaries? +/// +/// `.tmp..<8 hex>.marker`, with both middle parts checked. Matching on the prefix and +/// suffix alone would also take `.tmp.operator-notes.marker`, and this runs over a +/// directory holding a node's data, so what it removes is not a place to be approximate. +fn is_marker_temp_name(name: &str) -> bool { + let Some(rest) = name.strip_prefix(TEMP_PREFIX) else { + return false; + }; + let Some(rest) = rest.strip_suffix(".marker") else { + return false; + }; + let mut parts = rest.split('.'); + let (Some(pid), Some(nonce), None) = (parts.next(), parts.next(), parts.next()) else { + return false; + }; + !pid.is_empty() + && pid.bytes().all(|b| b.is_ascii_digit()) + && nonce.len() == 8 + && nonce.bytes().all(|b| b.is_ascii_hexdigit()) +} + +/// Remove marker temporaries a previous run left beside `path`. +/// +/// `write_file_atomic` writes its temporary next to its target. For the layout marker +/// that is inside `chunks/`, which the startup scan sweeps; for the migration marker it is +/// the node root, which nothing sweeps, so a crash between the write and the rename leaves +/// one there for the life of the node. Each is a few hundred bytes, so this is inodes +/// rather than capacity, but nothing else was ever going to remove them. +/// +/// Only the exact shape this module writes, and only files: a name has to carry the temp +/// prefix and the marker suffix. Anything broader would be this function deciding what +/// else in a node's root directory is rubbish, which is not its business. +/// +/// Best effort throughout. Failing to tidy up is not a reason to refuse to start, and the +/// caller takes the store lock before this runs, so there is no other process whose live +/// temporary this could take. +pub fn sweep_marker_temps(dir: &Path) { + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + for entry in entries.flatten() { + let name = entry.file_name(); + let Some(name) = name.to_str() else { + continue; }; - candidate.save(&self.config.root_dir)?; - *self.state.write() = candidate; - if shed == 0 { - info!("Committed to the file-backed key set; nothing has to be shed"); - } else { - info!( - "Committed to the file-backed key set; {shed} chunk(s) will be shed and \ - refetched once the legacy environment is gone and there is room" - ); + if !is_marker_temp_name(name) { + continue; + } + if !entry.file_type().is_ok_and(|kind| kind.is_file()) { + continue; + } + match std::fs::remove_file(entry.path()) { + Ok(()) => debug!( + "Swept a leftover marker temporary {}", + entry.path().display() + ), + Err(e) => debug!("Could not sweep {}: {e}", entry.path().display()), } - Ok(()) } +} - /// Record that the commitment builder has read and published the committable set. - /// - /// The retirement gate counts these: one proves the builder saw the new set, two - /// prove it survived a rotation, which is what makes the answerability window - /// meaningful rather than notional. - pub fn note_commitment_rebuilt(&self) { - let should_save = { - let mut state = self.state.write(); - if state.phase != MigrationPhase::Committed { - return; - } - if state.committed_at_unix.is_none() { - state.committed_at_unix = Some(crate::storage::migration::now_unix()); - } - state.rebuilds_since_commit = state.rebuilds_since_commit.saturating_add(1); - state.rebuilds_since_commit <= REQUIRED_REBUILDS_BEFORE_RETIRE - }; - if should_save { - let snapshot = self.state.read().clone(); - if let Err(e) = snapshot.save(&self.config.root_dir) { - warn!("Could not persist the migration marker: {e}"); +fn write_file_atomic(path: &Path, bytes: &[u8]) -> Result<()> { + let Some(dir) = path.parent() else { + return Err(Error::Storage(format!( + "Refusing to write {} — it has no parent directory", + path.display() + ))); + }; + let temp = dir.join(format!( + "{TEMP_PREFIX}{}.{:08x}.marker", + std::process::id(), + rand::random::() + )); + write_temp(&temp, bytes)?; + // Through the retry, because these small files (the layout marker, the migration + // state) are rewritten while the node runs, and on Windows a scanner holding a handle + // for a few milliseconds turns an ordinary rewrite into a hard failure. + rename_with_retry(&temp, path).map_err(|e| { + let _ = std::fs::remove_file(&temp); + Error::Storage(format!("Failed to publish {}: {e}", path.display())) + })?; + fsync_dir_best_effort(dir); + Ok(()) +} + +/// Read the layout marker, writing the current one if the store is new. +fn read_or_write_layout(chunks_dir: &Path) -> Result { + let path = chunks_dir.join(LAYOUT_FILE_NAME); + match read_small_file(&path) { + Ok(bytes) => serde_json::from_slice(&bytes).map_err(|e| { + Error::Storage(format!( + "Chunk store layout marker {} is unreadable: {e}. Refusing to open rather \ + than guess the layout.", + path.display() + )) + }), + Err(e) if e.kind() == ErrorKind::NotFound => { + if store_has_entries(chunks_dir) { + warn!( + "Chunk store at {} has data but no layout marker. Adopting it under \ + the current scheme, which is the only one this build implements. If \ + it was written by a build with a different layout its chunks will \ + appear to be missing.", + chunks_dir.display() + ); } + let layout = StoreLayout::default(); + let bytes = serde_json::to_vec_pretty(&layout) + .map_err(|e| Error::Storage(format!("Failed to encode chunk store layout: {e}")))?; + write_file_atomic(&path, &bytes)?; + debug!("Wrote chunk store layout marker to {}", path.display()); + Ok(layout) } + Err(e) => Err(Error::Storage(format!( + "Failed to read chunk store layout marker {}: {e}", + path.display() + ))), } +} - /// Whether every gate on deleting the legacy environment is satisfied. - /// - /// `still_answerable` is asked of each key the node is about to give up: the pruner's - /// existing retention contract, reused verbatim. A key still covered by a retained - /// commitment slot vetoes the delete, because the node could still be challenged on it. - pub fn retirement_blocker(&self, still_answerable: F) -> Option - where - F: Fn(&XorName) -> bool, +/// Whether the store directory already holds at least one shard. +fn store_has_entries(chunks_dir: &Path) -> bool { + let Ok(entries) = std::fs::read_dir(chunks_dir) else { + return false; + }; + entries.filter_map(std::result::Result::ok).any(|e| { + e.file_name() + .to_str() + .is_some_and(|n| n.len() == 2 && is_lower_hex(n)) + }) +} + +/// Largest a metadata marker may be before it is treated as corrupt. +const MAX_MARKER_BYTES: u64 = 64 * 1024; + +/// Read a small metadata file, refusing an implausibly large one. +/// +/// The chunk path is bounded for exactly this reason; the markers live in the same data +/// directory and deserve the same ceiling. +/// +/// # Errors +/// +/// Returns an I/O error, including `NotFound`, so callers can distinguish "no marker yet". +pub fn read_small_file(path: &Path) -> std::io::Result> { + let file = File::open(path)?; + let mut bytes = Vec::new(); + let read = file.take(MAX_MARKER_BYTES + 1).read_to_end(&mut bytes)?; + if read as u64 > MAX_MARKER_BYTES { + return Err(std::io::Error::other(format!( + "{} is larger than the {MAX_MARKER_BYTES} byte limit for a marker file", + path.display() + ))); + } + Ok(bytes) +} + +/// Take the store lock, or refuse to open the store. +/// +/// Both failures are refusals, deliberately. Unlike LMDB, which was genuinely +/// multi-process safe, two of these stores on one directory keep independent in-memory +/// indices, independent views of what is in flight, and independent opinions about +/// whether the legacy environment may be deleted: both would report the same write as +/// new and each would keep serving keys the other had deleted. A node that cannot create +/// the lock file has no way to know it is alone, and this is the one migration where +/// being wrong about that destroys data. +/// +/// The lock is an [`Arc`] so the work that relies on it can hold a lease. The startup +/// scan sweeps interrupted writes on the strength of being alone in the directory, and it +/// runs on a thread that outlives the future that started it. +/// +/// # Errors +/// +/// Returns [`Error::Storage`] when another process owns the directory, or when the lock +/// file cannot be created. +fn acquire_store_lock(chunks_dir: &Path) -> Result> { + let path = chunks_dir.join(LOCK_FILE_NAME); + let file = match OpenOptions::new() + .write(true) + .create(true) + .truncate(false) + .open(&path) { - // Before anything else, and whether or not there is a handle. An environment this - // node cannot classify must not be retired, and asking only on the no-handle path - // meant the ordinary path never asked: a node holding its store open went through - // every gate, renamed the directory aside and deleted it. - if self.legacy_cannot_be_classified() { - return Some(format!( - "{} cannot be read well enough to say whether it was already retired. \ - Nothing will be deleted until it can. Check that the directory and \ - anything inside it can be read.", - self.legacy_env_dir.display() - )); + Ok(f) => f, + // Not a warning and carry on. Without this lock two processes can open the same + // directory, each with its own index, its own view of what is in flight, and its + // own opinion about whether the legacy environment may be deleted. A node that + // cannot take it has no way to know it is alone, and this is the one migration + // where being wrong about that destroys data. + Err(e) => { + return Err(Error::Storage(format!( + "Could not create the chunk store lock {}: {e}. Refusing to start: \ + without it this node cannot tell whether another is using the same data \ + directory. Fix the permissions on that path, or remove a stale lock file \ + left by a different user.", + path.display() + ))) } - if !self.has_legacy() { - // No handle is not the same as no environment. A rename that failed and then - // could not be reopened leaves exactly that: the directory is still on disk - // and this node can no longer read it. Answering "nothing blocks retirement" - // would have the driver log the migration complete over a store that is still - // there and still holding chunks nothing else can serve. - let mark = retirement_mark(&self.legacy_env_dir); - if legacy_present(&self.config.root_dir).unwrap_or(true) && !mark.permits_removal() { - // Two different situations wearing one message would send an operator to - // the wrong place. One is a store this node cannot open; the other is a - // store nothing can even classify, which usually means a permission or a - // mount, and which the node deliberately will not act on either way. - if mark == RetirementMark::Unknown { - return Some(format!( - "{} is still on disk and this node cannot tell whether it was \ - retired, so it will neither open it nor remove it. Check that the \ - directory and anything inside it can be read.", - self.legacy_env_dir.display() - )); - } - return Some(format!( - "{} is still on disk but this node has no handle to it. It cannot be \ - read, verified or removed until the node is restarted.", - self.legacy_env_dir.display() - )); + }; + match file.try_lock_exclusive() { + Ok(()) => Ok(Arc::new(file)), + Err(e) => Err(Error::Storage(format!( + "Another process already has the chunk store at {} open ({e}). Two nodes \ + cannot share one data directory: each keeps its own index and they would \ + disagree about what is stored. Stop the other node first.", + chunks_dir.display() + ))), + } +} + +/// What a startup scan found. +struct ScanResult { + /// Every published address, ascending. + keys: Vec, + /// Which shard directories already exist. + shards_present: [bool; SHARD_COUNT], + /// Orphaned temp files removed. + swept_temps: usize, + /// Entries that were neither a chunk nor one of ours. + skipped: usize, +} + +/// Rebuild the key set from directory entries. +/// +/// Reads names only. A `stat` per entry costs about ten times the enumeration on Linux +/// and macOS and fifty to sixty times on Windows, and buys nothing: the filename is the +/// key, and the content is verified on read. +fn scan_store(chunks_dir: &Path) -> Result { + let mut result = ScanResult { + keys: Vec::new(), + shards_present: [false; SHARD_COUNT], + swept_temps: 0, + skipped: 0, + }; + + let top = std::fs::read_dir(chunks_dir).map_err(|e| { + Error::Storage(format!( + "Failed to enumerate chunk store {}: {e}", + chunks_dir.display() + )) + })?; + + for entry in top { + let entry = entry.map_err(|e| { + Error::Storage(format!( + "Failed to read an entry of {}: {e}", + chunks_dir.display() + )) + })?; + let name = entry.file_name(); + let Some(name) = name.to_str() else { + result.skipped = result.skipped.saturating_add(1); + continue; + }; + if name == LAYOUT_FILE_NAME || name == LOCK_FILE_NAME { + continue; + } + if name.starts_with(TEMP_PREFIX) { + if sweep_temp(&entry.path()) { + result.swept_temps = result.swept_temps.saturating_add(1); } - return None; + continue; } - if !self.config.migration.retire_legacy { - return Some( - "retirement is disabled in this release (storage.migration.retire_legacy)".into(), + if name.len() != 2 || !is_lower_hex(name) { + warn!( + "Chunk store: ignoring unexpected entry {name} in {}", + chunks_dir.display() ); + result.skipped = result.skipped.saturating_add(1); + continue; } - // A linked environment is never retired automatically. Retirement renames the - // path and then deletes what is behind it, and behind a link is a directory - // somewhere else that this node does not own. Copying still happens; only the - // removal is refused, so the node ends up serving from files with its old store - // intact and its operator told what to do about it. - if is_a_link(&self.legacy_env_dir) { - return Some(format!( - "{} is a link rather than a directory. The chunks are being copied out of \ - it, but it will not be deleted: what it points at is not this node's to \ - remove. Once the migration has settled, delete it by hand.", - self.legacy_env_dir.display() - )); - } - let state = self.state.read().clone(); - if state.phase != MigrationPhase::Committed { - return Some(format!("phase is {:?}, not Committed", state.phase)); - } - if state.rebuilds_since_commit < REQUIRED_REBUILDS_BEFORE_RETIRE { - return Some(format!( - "only {} of {REQUIRED_REBUILDS_BEFORE_RETIRE} commitment rebuilds observed", - state.rebuilds_since_commit - )); - } - if !state.retire_delay_elapsed(&self.config.migration) { - return Some(format!( - "the {}h retirement delay has not elapsed", - self.config.migration.effective_retire_delay_hours() - )); - } - if let Some(key) = self - .legacy_only_keys() - .into_iter() - .find(|k| still_answerable(k)) - { - return Some(format!( - "chunk {} is still answerable under a retained commitment", - hex::encode(key) - )); - } - None + let Ok(shard) = u8::from_str_radix(name, 16) else { + result.skipped = result.skipped.saturating_add(1); + continue; + }; + // `shards_present` is set inside `scan_shard`, on success only. Setting it from + // the name alone would make a stray regular file called `ab` look like a shard + // that already exists, and every write to that shard would then fail with a + // misleading error until the node was restarted. + scan_shard(&entry.path(), shard, &mut result)?; } - /// Re-hash every chunk that both stores hold, repairing the file from the legacy - /// copy when they disagree. - /// - /// A filename is not proof the bytes behind it are good. The startup scan reads - /// names only, so a file that was truncated or that rotted while the node was down is - /// indexed, counted as copied, committed to, and would have its intact legacy copy - /// deleted underneath it. The first verified read would then find the corruption with - /// nothing left to repair from. This pass is what turns "a file with that name - /// exists" into "those bytes are that chunk", and it is why it runs before - /// retirement rather than after. - /// - /// # Errors - /// - /// Returns [`Error::Storage`] if the legacy key set cannot be read. - pub async fn verify_before_retire( - &self, - throttle_mib_per_sec: u64, - shutdown: &CancellationToken, - ) -> Result { - let mut report = VerifyReport::default(); - // Taken BEFORE anything is read. Stamping it at the end would absorb exactly the - // failures this exists to catch: a chunk verified early in the pass that stops - // being readable before the pass finishes would leave the report carrying the - // already-incremented count, and both later checks would see it match. - let health_at_start = self.files.health_generation(); - let Some(legacy) = self.legacy() else { - report.ran = true; - report.health = health_at_start; - return Ok(report); - }; - report.ran = true; - - // Names before bytes. A chunk whose contents are durable but whose directory entry - // is not is still lost to a power loss, and the legacy copy is about to be deleted - // on the strength of this proof. Any failure here is a proof this pass did not - // produce. - if let Err(e) = self.files.flush_namespace() { - report.unrepairable = report.unrepairable.saturating_add(1); + result.keys.sort_unstable(); + result.keys.dedup(); + Ok(result) +} + +/// Scan one shard directory into `result`. +fn scan_shard(dir: &Path, shard: u8, result: &mut ScanResult) -> Result<()> { + let entries = match std::fs::read_dir(dir) { + Ok(e) => e, + // A stray file named like a shard, or a directory removed between the two reads. + // Neither is fatal, and neither marks the shard as present. + Err(e) if matches!(e.kind(), ErrorKind::NotFound | ErrorKind::NotADirectory) => { warn!( - "Could not make the file store's directory entries durable: {e}. The legacy \ - environment stays until they are." + "Chunk store: {} is not a shard directory ({e}); ignoring it", + dir.display() ); - return Ok(report); + result.skipped = result.skipped.saturating_add(1); + return Ok(()); + } + // Anything else is a real fault: a permission problem, exhausted descriptors, or + // failing hardware. Opening with a shard's worth of keys silently missing would + // make the node under-claim in its published commitment and stop serving chunks + // it still holds and is answerable for, so refuse to open at all. + Err(e) => { + return Err(Error::Storage(format!( + "Failed to enumerate shard {}: {e}. Refusing to open with an incomplete \ + key set.", + dir.display() + ))) } + }; + if let Some(slot) = result.shards_present.get_mut(shard as usize) { + *slot = true; + } - let legacy_keys = legacy.lmdb.all_keys().await?; - let total = legacy_keys.len(); - info!("Verifying {total} chunk(s) before removing the legacy environment"); - let mut since_log = 0u64; - for key in legacy_keys { - // This pass is a full read of the store and can run for hours. A shutdown - // must not wait it out, and an incomplete pass is simply not a clean proof. - if shutdown.is_cancelled() { - report.unrepairable = report.unrepairable.saturating_add(1); - debug!("Pre-retirement verification stopped for shutdown"); - return Ok(report); + for entry in entries { + let entry = + entry.map_err(|e| Error::Storage(format!("Failed to read {}: {e}", dir.display())))?; + let name = entry.file_name(); + let Some(name) = name.to_str() else { + result.skipped = result.skipped.saturating_add(1); + continue; + }; + if name.starts_with(TEMP_PREFIX) { + if sweep_temp(&entry.path()) { + result.swept_temps = result.swept_temps.saturating_add(1); } - // Under the key's critical section, so the two questions below are asked of - // one moment. Without it a write can publish the file and take the key out of - // the legacy-only set in between, and this pass would put it straight back. - let classified = { - let _lane = self.key_lock(&key).await; - // The physical question. A chunk the store holds but cannot currently - // read is still one it holds, and calling it absent here would put the - // key in the legacy-only set, where the union view advertises it again. - let in_files = self.files.is_indexed(&key); - let legacy_only = legacy.only.read().contains(&key); - if in_files && legacy_only { - // In both views at once, which nothing else clears once the copier - // has stopped running. The file store has it, so the legacy-only set - // is the one that is wrong: an answerable key in that set vetoes - // retirement for as long as the process lives. - debug!( - "Chunk {} was in both views; the file store has it, so it is no \ - longer legacy-only", - hex::encode(key) - ); - legacy.only.write().remove(&key); - } - (in_files, legacy_only) - }; - if !classified.0 { - // Known to be legacy-only, which is what a key this node is giving up - // looks like. Whether it may go is the gates' decision, not this pass's. - if classified.1 { - continue; - } - // In neither view. However that came about — a publish that failed, a - // file quarantined for corruption, a name this store stopped advertising — - // the environment holds the only copy, and nothing is looking after it: - // the gates only ever see the legacy-only set. Put it back there and - // refuse the proof this pass. What neither view protects is exactly what - // retirement destroys. + continue; + } + let Some(key) = decode_chunk_name(name) else { + if name.len() == CHUNK_NAME_LEN && is_hex_any_case(name) { + // A case-folded twin of a real chunk name. On NTFS and default APFS the + // existence check in the write path folds onto it, so a paid write would + // be answered "already stored" and its bytes dropped. Move it aside. + quarantine_entry(&entry.path()); + } else { warn!( - "Chunk {} is in the legacy environment, is not in the file store, and \ - was in neither view; re-queued for copying and the legacy environment \ - stays", - hex::encode(key) + "Chunk store: ignoring non-chunk entry {name} in {}", + dir.display() ); - legacy.only.write().insert(key); - report.unrepairable = report.unrepairable.saturating_add(1); - continue; } - since_log += 1; - if since_log >= VERIFY_LOG_EVERY { - since_log = 0; - info!( - "Pre-retirement verification: {} of at most {total} chunk(s) checked", - report.checked + result.skipped = result.skipped.saturating_add(1); + continue; + }; + // `file_type` comes from the directory entry itself on Linux and macOS and from + // the enumeration on Windows, so this is not the per-entry `stat` the scan + // deliberately avoids. A pipe, socket, device or directory wearing a chunk name + // must never enter the index: nothing downstream can read it, and it would sit in + // the published commitment forever. + match entry.file_type() { + Ok(kind) if kind.is_file() => {} + Ok(_) => { + warn!( + "Chunk store: {name} in {} is not a regular file; ignoring it", + dir.display() ); + result.skipped = result.skipped.saturating_add(1); + continue; } - let outcome = self.verify_one(&legacy, &key).await; - report.checked += 1; - report.bytes += outcome.bytes; - match outcome.verdict { - VerifyVerdict::Intact => {} - VerifyVerdict::Repaired => report.repaired += 1, - VerifyVerdict::Vanished => { - // The file went away while this pass was running, so the key is no - // longer file-backed. Put it back on the copier's list rather than - // republishing it here, where it could resurrect something the - // pruner deleted a moment ago. - legacy.only.write().insert(key); - report.unrepairable += 1; - } - VerifyVerdict::Unrepairable => report.unrepairable += 1, - } - if let Some(delay) = throttle_delay(outcome.bytes, throttle_mib_per_sec) { - tokio::time::sleep(delay).await; + // Not the same as knowing it is not a file. Treating an unanswered question + // as a no would drop a real chunk from the index and from the commitment + // while its bytes sit on disk, and the node would not serve it again until + // some later restart happened to succeed. Fail the scan instead: an index + // that is missing keys must never be published as this node's key set. + Err(e) => { + return Err(Error::Storage(format!( + "Could not tell what {name} in {} is: {e}. Refusing to publish an \ + index that may be missing chunks.", + dir.display() + ))); } } - - if report.unrepairable == 0 { - info!( - "Pre-retirement verification passed: {} chunk(s) checked, {} repaired", - report.checked, report.repaired - ); - } - // The count this pass started from, and a refusal if the store moved while it - // ran. Retirement compares the same value again immediately before deleting - // anything, so one number covers both windows: during the pass, and after it. - report.health = health_at_start; - if self.files.health_generation() != health_at_start { + // A file in the wrong shard is unreachable through `chunk_path`, so indexing it + // would make the index claim a key the read path cannot find. + if shard_index(&key) != shard as usize { warn!( - "A chunk stopped being servable while the pre-retirement pass was running, \ - so this pass does not describe the store. Another runs on the next tick." + "Chunk store: {name} is filed under shard {shard:02x} but belongs in {:02x}; \ + ignoring it. Move it or delete it.", + shard_index(&key) ); - report.unrepairable = report.unrepairable.saturating_add(1); - } - Ok(report) - } - - /// Check one chunk that both stores hold, repairing the file if it is wrong. - async fn verify_one(&self, legacy: &Legacy, key: &XorName) -> VerifyOutcome { - // The throttle sleep is deliberately outside this critical section: at 32 MiB/s a - // 4 MiB chunk sleeps for over a tenth of a second, and holding a shard lane for - // that would stall every write to a sixteenth of the address space for hours. - let _lane = self.key_lock(key).await; - - let bytes = match self.files.get_raw(key).await { - Ok(bytes) => bytes, - // Not the same as gone. `Vanished` puts the key back on the copier's list, - // and doing that for a file that is still there and still indexed leaves the - // key in both views at once: the file index keeps it in every commitment, so - // it stays answerable, and an answerable legacy-only key vetoes retirement for - // as long as the process lives. Refuse this pass instead. - Err(e) => { - warn!( - "Chunk {} could not be read while verifying: {e}. The legacy \ - environment stays.", - hex::encode(key) - ); - return VerifyOutcome { - bytes: 0, - verdict: VerifyVerdict::Unrepairable, - }; - } - }; - let len = bytes.as_ref().map_or(0, Vec::len) as u64; - let Some(bytes) = bytes else { - return VerifyOutcome { - bytes: 0, - verdict: VerifyVerdict::Vanished, - }; - }; - if crate::client::compute_address(&bytes) == *key { - // The pass hashed these bytes and they are right, so whatever this store - // thought was wrong with them is not. It reads raw, which does not settle - // that on its own, and leaving the mark would retire the environment while a - // healthy chunk stayed unadvertised until some later verified read. - self.files.note_bytes_proven_good(key); - return VerifyOutcome { - bytes: len, - verdict: VerifyVerdict::Intact, - }; + result.skipped = result.skipped.saturating_add(1); + continue; } + result.keys.push(key); + } + Ok(()) +} - warn!( - "Chunk {} is in the file store but does not match its address; rewriting it \ - from the legacy environment before that environment is removed", - hex::encode(key) - ); - // Replace in place. Deleting first and writing after would leave a window whose - // only surviving copy is the one this whole pass exists to make safe to delete. - let verdict = match legacy.lmdb.get_raw(key).await { - Ok(Some(good)) if self.files.repair(key, &good).await.is_ok() => { - VerifyVerdict::Repaired - } - _ => { - warn!( - "Chunk {} could not be rewritten from the legacy environment. \ - Retirement stays blocked so its bytes are not thrown away.", - hex::encode(key) - ); - VerifyVerdict::Unrepairable - } - }; - VerifyOutcome { - bytes: len, - verdict, +/// Remove one orphaned temp file. Returns whether it went. +/// +/// Always removed. The scan that calls this runs only after the store lock has been taken, +/// so by then any temp file is an interrupted write of a previous run and there is no other +/// process that could be writing it. This used to describe a second, gentler mode for the +/// unlocked case; there was never any such branch and there is no caller that would need +/// one. +fn sweep_temp(path: &Path) -> bool { + match std::fs::remove_file(path) { + Ok(()) => { + debug!("Removed orphaned temporary file {}", path.display()); + true + } + Err(e) => { + debug!("Could not remove {}: {e}", path.display()); + false } } +} - /// Is this verification still worth acting on? - /// - /// # Errors - /// - /// Returns [`Error::Storage`] naming what has changed since the pass ran. - fn proof_is_usable(&self, proof: &VerifyReport) -> Result<()> { - if !proof.is_clean() { +/// Open a chunk file, refusing anything that is not a regular file. +/// +/// `Ok(None)` means the file is not there. A named pipe wearing a valid chunk name would +/// otherwise block the opening thread forever: `open` on a FIFO with no writer does not +/// return, and enough of them would exhaust the blocking pool and stall every file and +/// database operation in the process. `O_NOFOLLOW` refuses a symlink for the same reason, +/// and both are checked on the handle rather than the path, so nothing can be swapped +/// underneath between the check and the open. +fn open_regular(path: &Path) -> Result> { + #[cfg(unix)] + let opened = { + use std::os::unix::fs::OpenOptionsExt; + OpenOptions::new() + .read(true) + .custom_flags(libc::O_NONBLOCK | libc::O_NOFOLLOW) + .open(path) + }; + #[cfg(not(unix))] + let opened = OpenOptions::new().read(true).open(path); + + let file = match opened { + Ok(f) => f, + Err(e) if e.kind() == ErrorKind::NotFound => return Ok(None), + Err(e) => { return Err(Error::Storage(format!( - "Refusing to remove the legacy environment: verification reported {} \ - unrepairable chunk(s) (ran: {})", - proof.unrepairable, proof.ran - ))); + "Failed to open chunk file {}: {e}", + path.display() + ))) } - if !proof.still_describes(&self.files) { - return Err(Error::Storage( - "Refusing to remove the legacy environment: a chunk stopped being \ - servable since it was verified, so that verification no longer describes \ - the file store. A fresh pass runs on the next tick." - .into(), - )); - } - if self.has_pending_writes() { - return Err(Error::Storage( - "Refusing to remove the legacy environment: a write announced itself and \ - has not reported back, so what the environment holds is not yet settled." - .into(), - )); + }; + let is_regular = file.metadata().is_ok_and(|m| m.file_type().is_file()); + if !is_regular { + return Err(Error::Storage(format!( + "{} is not a regular file; refusing to read it as a chunk", + path.display() + ))); + } + Ok(Some(file)) +} + +/// Read a chunk file, refusing anything larger than a chunk can legitimately be. +/// +/// A corrupt, sparse, or locally planted file wearing a valid 64-hex name would +/// otherwise be read straight into memory, so a single bad entry could exhaust the node +/// during an ordinary GET or an audit response. +fn read_bounded(file: File, path: &Path) -> Result> { + let ceiling = MAX_CHUNK_SIZE as u64; + let mut buf = Vec::new(); + let read = file.take(ceiling + 1).read_to_end(&mut buf).map_err(|e| { + Error::Storage(format!("Failed to read chunk file {}: {e}", path.display())) + })?; + if read as u64 > ceiling { + return Err(Error::Storage(format!( + "Chunk file {} is larger than the {ceiling} byte maximum; refusing to read it", + path.display() + ))); + } + Ok(buf) +} + +/// Whether a Windows error is one a scanner or indexer holding a handle would produce. +/// +/// `ERROR_ACCESS_DENIED`, `ERROR_SHARING_VIOLATION`, `ERROR_LOCK_VIOLATION`. Every other +/// failure is deterministic and retrying it only burns a blocking thread. +fn is_windows_sharing_violation(e: &std::io::Error) -> bool { + matches!(e.raw_os_error(), Some(5 | 32 | 33)) +} + +/// Publish `temp_path` as `final_path`, retrying a transient sharing violation. +/// +/// On Windows an antivirus scanner or the search indexer can hold a handle to either +/// file for a few milliseconds after it is created, and `MoveFileEx` fails outright +/// rather than queueing. Retrying a bounded number of times turns that from a failed +/// write into a short pause. Every other error returns immediately. +/// +/// # Errors +/// +/// Returns the rename's error: at once for anything but a Windows sharing violation, and +/// the last one after the bounded retries for that. +pub fn rename_with_retry(temp_path: &Path, final_path: &Path) -> std::io::Result<()> { + let mut last = match std::fs::rename(temp_path, final_path) { + Ok(()) => return Ok(()), + Err(e) => e, + }; + if !cfg!(windows) || !is_windows_sharing_violation(&last) { + return Err(last); + } + for attempt in 1..=RENAME_RETRY_ATTEMPTS { + std::thread::sleep(RENAME_RETRY_BACKOFF * attempt); + match std::fs::rename(temp_path, final_path) { + Ok(()) => return Ok(()), + Err(e) => last = e, } - Ok(()) } + Err(last) +} - /// Close the legacy environment and remove it, returning the bytes freed. - /// - /// This is the only destructive step in the migration and the only one that cannot - /// be undone. It is also the only moment the disk comes back. - /// - /// Takes a [`VerifyReport`] rather than a flag so the verification pass cannot be - /// skipped: there is no way to call this without having produced one. - /// - /// # Errors - /// - /// Returns [`Error::Storage`] if verification did not pass or no longer describes the - /// store, if a write has not reported back, if the handle is still shared (the caller - /// should retry on the next tick), or if the directory cannot be removed. - pub async fn retire_legacy( - &self, - proof: &VerifyReport, - still_answerable: &F, - approved_to_shed: &BTreeSet, - ) -> Result - where - F: Fn(&XorName) -> bool + Send + Sync, +/// Write `payload` and publish it as `final_path`, replacing whatever is there. +/// +/// Success here means the bytes are durable, not merely written. The repair path this +/// serves runs during the pre-retirement pass, where a chunk that fails to match its +/// address is rewritten from the legacy store and the legacy store is then deleted. A +/// replacement that a power loss can undo would leave that chunk with the wrong bytes and +/// no other copy. +fn write_and_replace( + temp_path: &Path, + final_path: &Path, + payload: &[u8], + shard: &Path, +) -> Result<()> { + // Unix: an intra-directory rename is atomic, so a reader sees the old content or the + // new one and never an absence, and the directory flush is what makes it durable. + #[cfg(unix)] { - self.proof_is_usable(proof)?; - // Rechecked here, not only by the caller. Everything between the caller's check - // and this point is a window: the verification pass alone can run for hours, and - // a write whose file half failed inserts a new legacy-only key in the meantime. - if let Some(reason) = self.retirement_blocker(still_answerable) { - return Err(Error::Storage(format!( - "Refusing to remove the legacy environment: {reason}" - ))); - } - // Exclusive from here until the handle is out. Every read, write and delete holds - // this shared, so taking it means none is in progress and none can start: no - // reader is mid-way between discarding a corrupt file and reaching the copy that - // would replace it, and nothing new can start work in an environment that is about - // to go idle. - // - // Released as soon as the handle has been taken and the directory renamed away, - // which is the point after which nothing can reach the environment anyway. The - // deletion that follows can take a long time on a large store, and holding every - // chunk request on the node behind it would turn retirement into an outage. - // - let retiring = self.retirement.write().await; - // Asked again with the guard held, which is the only moment the answer cannot - // change underneath it. The check above can be overtaken by a read that fails - // between there and here. - if !proof.still_describes(&self.files) { - drop(retiring); - return Err(Error::Storage( - "Refusing to remove the legacy environment: a chunk stopped being \ - servable while retirement was starting. A fresh pass runs on the next \ - tick." - .into(), - )); - } - let Some(legacy) = self.legacy() else { - return Ok(0); - }; - let freed = self.legacy_bytes(); - // Let go of our own clone straight away, so the only strong reference that should - // remain is the one the store itself holds. - drop(legacy); - - for attempt in 0..RETIRE_UNWRAP_ATTEMPTS { - // Drained on every attempt, not once up front: `LmdbStorage`'s blocking - // closures capture a cloned `Env` rather than the `Arc`, so the strong count - // alone would not notice a read that is still mapped. The tracker does, and - // it reopens itself, so a read that started since the last drain needs - // another one. - if let Some(l) = self.legacy() { - l.lmdb.wait_idle().await; - drop(l); - } - // Taking the handle out and proving sole ownership happen in the same - // critical section. Deliberately not two steps: taking it first and putting - // it back on failure would leave a window in which reads see no legacy store - // and report a chunk that lives only there as missing. - let taken = { - let mut guard = self.legacy.write(); - match guard.as_ref() { - // A strong count of one means nobody else holds a handle, so nobody - // can be reading the legacy store *or* mutating its key set. That is - // what makes the final check below atomic with the removal: this is - // the only moment at which the answer cannot change underneath us. - Some(l) if Arc::strong_count(&l.lmdb) == 1 => { - // Asked here, in the same critical section as the checks below - // and immediately before the handle is taken. Asking earlier is - // not enough: a write can announce itself under the shared guard, - // be cancelled so the guard is released, and leave its blocking - // half running past the drain above. Its note is the only thing - // that says so, and dropping the journal with the environment - // would take the evidence with it. - if !l.pending.read().is_empty() { - return Err(Error::Storage( - "Refusing to remove the legacy environment: a write \ - announced itself and has not reported back, so what the \ - environment holds is not yet settled." - .into(), - )); - } - let only = l.only.read(); - // A count of one proves nobody else holds a handle, so nobody can - // be mutating this set. That is what makes the two checks below - // authoritative rather than a snapshot that has already moved. - if let Some(key) = only.iter().find(|k| still_answerable(k)) { - return Err(Error::Storage(format!( - "Refusing to remove the legacy environment: chunk {} became \ - answerable again while retirement was in progress", - hex::encode(key) - ))); - } - // Only the keys the caller cleared may go. A write whose file half - // failed adds a legacy-only key that is in no commitment, so the - // answerability check above cannot see it, and it would otherwise - // be destroyed without ever facing the rank, delivery or - // possession gates. - if let Some(key) = only.iter().find(|k| !approved_to_shed.contains(*k)) { - return Err(Error::Storage(format!( - "Refusing to remove the legacy environment: chunk {} entered \ - the legacy-only set after the gates were cleared and has \ - passed none of them", - hex::encode(key) - ))); - } - drop(only); - guard.take() - } - Some(_) => None, - None => return Ok(0), - } - }; - if let Some(Legacy { - lmdb, - only, - pending, - skipped_rollback_copies, - }) = taken - { - drop(only); - drop(pending); - drop(skipped_rollback_copies); - drop(lmdb); - return self.remove_legacy_dir(freed, retiring).await; - } - if attempt + 1 < RETIRE_UNWRAP_ATTEMPTS { - tokio::time::sleep(RETIRE_UNWRAP_BACKOFF).await; - } - } - - // Nothing was taken and nothing will be this tick, so let the node get on with - // serving rather than leaving this held until the function returns. - drop(retiring); - Err(Error::Storage( - "Legacy environment is still being read; retirement deferred to the next tick".into(), - )) - } - - /// Remove the legacy directory and record that the migration is over. - /// - /// The handle is already closed by the time this runs, so the node is file-only - /// either way. If the removal fails the phase still moves on, because there is no - /// going back to a half-removed environment, and the operator is told exactly which - /// directory to delete by hand to get the space back. - async fn remove_legacy_dir( - &self, - freed: u64, - retiring: tokio::sync::RwLockWriteGuard<'_, ()>, - ) -> Result { - // Renamed aside first, because `remove_dir_all` is not atomic: a failure partway - // through leaves a directory that can no longer be opened as an environment, and - // recording the migration as finished on top of that would have the node claim - // completion over a half-deleted store. A rename either happens or does not. - let tombstone = free_tombstone_path(&self.config.root_dir); - - if let Err(e) = std::fs::rename(&self.legacy_env_dir, &tombstone) { - // Nothing was deleted, but the handle is already closed, so this node has - // stopped being able to serve anything that lives only in there. Put it back - // rather than carrying on with chunks it holds and cannot read, and rather - // than letting the next tick see no handle and call that success. - let restored = self.reopen_legacy().await; - return Err(Error::Storage(format!( - "Could not move the legacy environment {} aside: {e}. Nothing was deleted{}", - self.legacy_env_dir.display(), - if restored { - " and it has been reopened, so the node keeps serving from both stores." - } else { - ". IT COULD NOT BE REOPENED: this node cannot serve chunks that live only there until it is restarted." - } - ))); - } - // The rename has to reach the directory itself, not just the page cache, and this - // one is not best effort. The tombstone is deleted a few lines below. If the - // rename has not reached the disk when that happens, a power loss brings the - // environment back under its old name with its contents already removed, and the - // next start finds a corrupt environment it cannot open. Stopping here instead - // leaves the tombstone in place, which the next start sweeps. - // Marked from the inside, now that the rename has succeeded and before anything - // is deleted. This is what a directory that reverts to its old name carries with - // it, and it is the only thing a later start treats as permission to delete. - if let Err(e) = mark_directory_retired(&tombstone) { - // Nothing has been deleted and the directory is intact, so put it back rather - // than recording the migration as finished over a store that is still there. - // Recording finished would be worse than it sounds: the next tick restores the - // unmarked directory to its own name, and a node that has already called - // itself file-only would then exit with a live environment on disk and no - // handle to it. - // Only when the mark is provably gone. A mark left inside would have the - // next cleanup pass reap a live, open environment. - let restored = - e.mark_definitely_gone && std::fs::rename(&tombstone, &self.legacy_env_dir).is_ok(); - let reopened = restored && self.reopen_legacy().await; + write_temp(temp_path, payload)?; + if let Err(e) = rename_with_retry(temp_path, final_path) { + let _ = std::fs::remove_file(temp_path); return Err(Error::Storage(format!( - "Moved the legacy environment to {} but could not mark it retired: {e}. \ - Nothing was deleted{}", - tombstone.display(), - if reopened { - ", and it has been put back, so the node keeps serving from both \ - stores and retirement is tried again." - } else if e.mark_definitely_gone { - ". IT COULD NOT BE PUT BACK: this node cannot serve chunks that live \ - only there until it is restarted." - } else { - ". It has been left where nothing will open it, because a partial \ - retirement mark may still be inside it. Its chunks are in the file \ - store; move it back by hand only after removing that mark." - } + "Failed to replace chunk {}: {e}", + final_path.display() ))); } - - // Test-only: renamed aside and marked, nothing deleted yet. A process killed here - // is what the recovery on the next start exists for. - #[cfg(any(test, feature = "test-utils"))] - crate::storage::file_store::halt_here_if_asked( - crate::storage::file_store::HALT_AFTER_RETIRE_MARK, - &tombstone, - ); - - if let Err(e) = crate::storage::file_store::fsync_path(&self.config.root_dir) { - warn!( - "The legacy environment was moved aside but {} could not be flushed: {e}. \ - Leaving {} in place rather than deleting a directory whose new name may \ - not have reached the disk. The next start finishes this.", - self.config.root_dir.display(), - tombstone.display() - ); - self.finish_migration(); - return Ok(0); - } - self.finish_migration(); - - // From here nothing can reach the environment: its handle is gone and its - // directory is under a name no code looks for. Let the node serve again rather - // than holding every chunk request behind a deletion that can run for minutes. - drop(retiring); - - // Only now, and best effort: the bytes come back when this completes, and if it - // does not the next start sweeps the tombstone. - // - // On a detached OS thread, and not awaited. This is a synchronous recursive delete - // of a directory that can hold hundreds of gigabytes and cannot be interrupted - // once it starts. Inside the migration task it would sit through shutdown's grace - // and past it, because an abort is not observed until the call returns; on the - // runtime's blocking pool a normal runtime shutdown would wait for it anyway. A - // plain thread is the only one the process can genuinely walk away from, and the - // directory carries its own retirement mark, so whatever is left is finished by - // the next start. - delete_retired_directory(tombstone); - Ok(freed) - } - - /// Try again to open a legacy environment this node has lost its handle to. - /// - /// A rename that failed and then could not be reopened leaves the directory on disk - /// with no way to read it, and every chunk that lives only there unserved. Saying so - /// once and waiting for a restart is not enough: the reason is usually transient, and - /// a node that is otherwise healthy should not stay half-blind until somebody notices. - /// - /// Returns whether it came back. Does nothing when there is a handle already, or when - /// there is nothing on disk to open. - pub async fn recover_lost_legacy_handle(&self) -> bool { - if self.has_legacy() || !retirement_mark(&self.legacy_env_dir).permits_opening() { - return false; - } - if !legacy_present(&self.config.root_dir).unwrap_or(false) { - return false; - } - // Opened WITHOUT the exclusive guard. Opening scans every key in the environment, - // which on a large store is minutes, and every read and write on the node would - // wait behind it. Nothing else can be installing a handle: retirement does nothing - // while there is none, and this runs from the one migration task. - let (lmdb, legacy_keys) = match Self::open_legacy_env(&self.config).await { - Ok(opened) => opened, - Err(e) => { - warn!( - "Could not reopen {}: {e}. The chunks that live only there stay \ - unreadable until this succeeds.", - self.legacy_env_dir.display() - ); - return false; - } - }; - // Exclusive only to install it, which is instant. - let _recovering = self.retirement.write().await; - if self.has_legacy() { - return false; - } - // The diff happens HERE, not when the environment was read. Reading it takes - // minutes on a large store, and a verifying read in that time can find a file - // rotted and throw it away. With no handle installed there was nothing to put the - // key back into, so a set computed beforehand would be missing it, every gate - // would skip it, and retirement would destroy the intact copy in the environment. - // Under this guard no read, write or delete is in flight, so the file store's - // answer cannot move while it is being asked. - let only = Self::keys_only_in_legacy(&legacy_keys, &self.files); - *self.legacy.write() = Some(Legacy { - lmdb, - only: Arc::new(parking_lot::RwLock::new(only)), - skipped_rollback_copies: Arc::new(std::sync::atomic::AtomicU64::new(0)), - pending: Arc::new(parking_lot::RwLock::new(BTreeMap::new())), - }); - // A node that recorded itself file-only and then got an environment back has to - // go through the migration again from the start: the phase decides what the - // driver does, and file-only does no copying, so leaving it there would give the - // node a handle it never uses. Conservative on purpose; the copier finds most of - // the work already done. - if self.migration_phase() == MigrationPhase::FilesOnly { - warn!( - "Recovered a legacy chunk environment after recording this node as \ - file-only. Starting the migration again from the copying stage." - ); - let mut state = self.state.write(); - state.phase = MigrationPhase::Bridging; - state.committed_at_unix = None; - state.rebuilds_since_commit = 0; - let snapshot = state.clone(); - drop(state); - if let Err(e) = snapshot.save(&self.config.root_dir) { - warn!("Could not persist the migration marker: {e}"); - } - } - warn!( - "Reopened {} after losing its handle", - self.legacy_env_dir.display() - ); - true - } - - /// Is there a retired directory still waiting to be deleted? - /// - /// Separate from having a legacy environment: a node whose removal was interrupted has - /// no handle and nothing to migrate, but its disk has not come back. Something has to - /// keep trying during this uptime rather than leaving it until the next restart. - #[must_use] - pub fn has_cleanup_pending(&self) -> bool { - !retired_tombstones(&self.config.root_dir).is_empty() - || !retirement_mark(&self.legacy_env_dir).permits_opening() - } - - /// Try again to finish a removal a previous attempt left behind. - /// - /// Safe to call at any time: it only ever moves or deletes a directory that carries - /// its own retirement mark. - pub fn retry_cleanup(&self) { - // Never while this node has the environment open. Cleanup decides what to do from - // the directory's own mark, and a mark that outlived a failed retirement would - // have it rename a live, mapped environment out from under the handle. - if self.has_legacy() { - sweep_retired_legacy(&self.config.root_dir); - return; - } - finish_interrupted_retirement(&self.config.root_dir); - } - - /// Resolve writes whose outcome was never recorded. - /// - /// A write announces itself before it starts and clears the note when both halves - /// have returned. A note still there afterwards belongs to a write nobody waited for, - /// and only the disk can say what became of it: the file store has the chunk, or the - /// environment does and nothing else, or neither and there was never anything to - /// protect. - pub async fn reconcile_pending_writes(&self) { - if !self.has_pending_writes() { - return; - } - // Exclusively, and before the snapshot. Draining is not a barrier on its own: - // writes hold this shared, and a new one for the same key could announce itself, - // be cancelled, and leave its blocking half running while this decided the older - // one's fate and removed the single entry they share. Held here, nothing new can - // start, so what the disk says once the drain returns is final. - // - // Only reached when something is waiting, which after a clean run is never, so - // this is not a stall on the ordinary path. - let _settling = self.retirement.write().await; - let Some(legacy) = self.legacy() else { - return; - }; - let waiting: Vec = legacy.pending.read().keys().copied().collect(); - if waiting.is_empty() { - return; - } - legacy.lmdb.wait_idle().await; - self.files.wait_idle().await; - for key in waiting { - let _lane = self.key_lock(&key).await; - if self.files.is_indexed(&key) { - legacy.only.write().remove(&key); - legacy.pending.write().remove(&key); - continue; - } - match legacy.lmdb.get_raw(&key).await { - Ok(Some(_)) => { - debug!( - "Chunk {} was written to the legacy environment by a call that \ - never returned; recording it so the copier picks it up", - hex::encode(key) - ); - legacy.only.write().insert(key); - legacy.pending.write().remove(&key); - } - // Nothing behind it: there was never anything to protect. - Ok(None) => { - legacy.pending.write().remove(&key); - } - // NOT the same as nothing behind it. Dropping the note on a read that - // failed would leave a committed write with no protection at all, which - // is the case this journal exists for. Keep it and ask again next tick; - // retirement stays vetoed meanwhile. - Err(e) => warn!( - "Could not tell what became of the write for {}: {e}. Asking again on \ - the next tick.", - hex::encode(key) - ), - } - } - } - - /// Are there writes in flight whose outcome nothing has recorded? - #[must_use] - pub fn has_pending_writes(&self) -> bool { - self.legacy().is_some_and(|l| !l.pending.read().is_empty()) - } - - /// Is there anything at the legacy environment's path at all? - /// - /// Asked without a handle, and answered conservatively: a path this node cannot even - /// look at counts as present. The migration is not finished while something is there, - /// whether or not this node can currently read it. - #[must_use] - pub fn legacy_dir_is_on_disk(&self) -> bool { - // `symlink_metadata`, not `try_exists`, which follows links. An operator's link to - // storage that is not mounted right now reads as nothing at all through the - // second, and the node would call its migration finished and go file-only, blind - // to every chunk that lives only there until somebody restarts it. - match std::fs::symlink_metadata(&self.legacy_env_dir) { - Ok(_) => true, - // Only "it is not there" means it is not there. A permission change or a - // transient fault is an unanswered question, and answering it with "nothing - // here" is how the driver declares the migration finished over a store it has - // merely lost sight of. - Err(e) => e.kind() != std::io::ErrorKind::NotFound, - } - } - - /// Is the legacy environment a link this node must not delete? - /// - /// Copying out of it works; only the removal is refused. Callers use this to stop - /// waiting for a retirement that is never going to happen. - #[must_use] - pub fn legacy_is_a_link(&self) -> bool { - self.has_legacy() && is_a_link(&self.legacy_env_dir) - } - - /// How many writes this node made without a rollback copy, cumulatively. - /// - /// Zero on a node that is not bridging, and zero on a bridging node whose environment - /// has room. A number that is climbing says this node would lose those chunks on a - /// rollback to a pre-migration build, which is a fleet question the second release - /// turns on and which a per-chunk log line cannot answer. - /// - /// Attempts rather than distinct chunks, for the reason given on the field: it is a - /// rate, not an inventory. - #[must_use] - pub fn writes_without_a_rollback_copy(&self) -> u64 { - self.legacy().map_or(0, |l| { - l.skipped_rollback_copies - .load(std::sync::atomic::Ordering::Relaxed) - }) - } - - /// Is there an environment on disk this node cannot classify at all? - /// - /// Neither removable nor openable, which is not a state waiting will clear: something - /// about the path has to change first, and until it does the node will refuse to touch - /// it in either direction. The driver treats this the way it treats a lost handle or a - /// link, by standing down from the shared volume and saying so where an operator looks, - /// because holding a disk exclusively to wait for a person is a disk nobody else can - /// use. - #[must_use] - pub fn legacy_cannot_be_classified(&self) -> bool { - retirement_mark(&self.legacy_env_dir) == RetirementMark::Unknown - } - - /// Is there an environment on disk this node can no longer read? - #[must_use] - pub fn has_lost_its_legacy_handle(&self) -> bool { - !self.has_legacy() - && retirement_mark(&self.legacy_env_dir).permits_opening() - // Conservative in the same direction as the retirement blocker, which reads the - // same failure as "there is one". A question that cannot be answered is not an - // answer of no, and answering no here left the node holding the shared volume - // for the six-hour cap over work no amount of disk will finish. - && legacy_present(&self.config.root_dir).unwrap_or(true) - } - - /// Reopen the legacy store after a failed retirement, so the node keeps serving. - /// - /// Returns whether it came back. The handle is closed before the rename is attempted, - /// so a rename that fails leaves the node holding chunks it can no longer read; that - /// is worth undoing rather than living with until the next restart. - async fn reopen_legacy(&self) -> bool { - if !legacy_present(&self.config.root_dir).unwrap_or(false) { - return false; - } - match Self::open_legacy(&self.config, &self.files).await { - Ok(legacy) => { - *self.legacy.write() = Some(legacy); - warn!("Reopened the legacy chunk environment after a failed retirement"); - true - } - Err(e) => { - error!("Could not reopen the legacy chunk environment: {e}"); - false - } - } + fsync_dir(shard).map_err(|e| { + Error::Storage(format!( + "Replaced {} but could not flush {}: {e}. Not reporting the repair as \ + done, because a rewrite that is not durable must not authorise deleting \ + the copy it was rewritten from.", + final_path.display(), + shard.display() + )) + })?; + Ok(()) } - - /// Record that this node serves from files alone from here on. - fn finish_migration(&self) { - self.files.invalidate_capacity_cache(); - { - let mut state = self.state.write(); - state.phase = MigrationPhase::FilesOnly; - } - let snapshot = self.state.read().clone(); - if let Err(e) = snapshot.save(&self.config.root_dir) { - warn!("Could not persist the migration marker: {e}"); - } + // Everywhere else, Windows included: there is no way to flush a directory through the + // standard library, so a rename cannot be shown to be durable at return. Overwriting + // the existing file changes no directory entry at all, and `sync_all` (FlushFileBuffers + // on Windows) is documented to flush the file's data, so a successful return is + // durable under a documented contract. + // + // The cost is that this is not atomic: a crash part-way leaves the file holding a mix + // of old and new bytes. + // + // That used to be justified by the legacy store still being there to repair from, which + // it no longer is. The argument now is narrower and does not depend on a second copy: + // every caller reaches this only after a read has proven the bytes under that name + // wrong. A crash part-way therefore leaves wrong bytes where wrong bytes already were, + // which is not a loss, and the next verified read finds them and repairs again. What it + // is NOT safe for is replacing bytes that were good, so this must not be reached on any + // path that has not established otherwise. In this crate that holds: the two + // `StoredBytes::Wrong` arms get there from a read that hashed and disagreed, and + // `holds_verified` gets there from a read that returned bytes which were not the + // caller's. The public entry point makes no such check and says so. + // + // A temporary and a rename would make it atomic, at the cost of a directory entry + // change that cannot be flushed here. That trade is worth revisiting on a platform + // where it can actually be tested; it is not worth making blind. + #[cfg(not(unix))] + { + let _ = temp_path; + let _ = shard; + let mut file = OpenOptions::new() + .write(true) + .truncate(true) + .open(final_path) + .map_err(|e| { + Error::Storage(format!( + "Failed to open {} for replacement: {e}", + final_path.display() + )) + })?; + file.write_all(payload).map_err(|e| { + Error::Storage(format!("Failed to rewrite {}: {e}", final_path.display())) + })?; + file.sync_all().map_err(|e| { + Error::Storage(format!( + "Rewrote {} but could not flush it: {e}. Not reporting the repair as \ + done, because a rewrite that is not durable must not authorise deleting \ + the copy it was rewritten from.", + final_path.display() + )) + })?; + Ok(()) } } -/// What checking one chunk concluded. -enum VerifyVerdict { - /// The file matches its name. - Intact, - /// The file was wrong and was rewritten from the legacy copy. - Repaired, - /// The file was wrong and could not be rewritten. - Unrepairable, - /// The file disappeared while the pass was running. - Vanished, -} - -/// One chunk's verification result. -struct VerifyOutcome { - /// Bytes read, for the throttle. - bytes: u64, - /// What was concluded. - verdict: VerifyVerdict, -} - -/// What the pre-retirement verification pass found. +/// Create `temp_path`, write `payload` into it, and flush it. /// -/// Every field is private, and the only way to obtain one is -/// [`ChunkStore::verify_before_retire`]. That is deliberate: it is the sole evidence -/// [`ChunkStore::retire_legacy`] accepts that the file store really holds what it claims, -/// and a report anyone could construct would be no evidence at all. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -pub struct VerifyReport { - /// Whether the pass actually ran. - ran: bool, - /// Chunks re-hashed. - checked: u64, - /// Bytes read. - bytes: u64, - /// Chunks whose file was wrong and was rewritten from the legacy copy. - repaired: u64, - /// Chunks whose file was wrong and could not be repaired. - unrepairable: u64, - /// What the file store's health looked like when this pass finished. - /// - /// A clean report is reused for a while rather than re-read on every tick, and a lot - /// can happen in that window: a kept file can start failing to read while ordinary - /// requests are served from the legacy copy, and the node would then delete the - /// legacy copy on the strength of a pass that no longer describes the store. This is - /// how retirement tells, immediately before it deletes anything. - health: u64, -} - -impl VerifyReport { - /// Whether this report clears the way for retirement. - #[must_use] - pub fn is_clean(&self) -> bool { - self.ran && self.unrepairable == 0 - } - - /// Does this report still describe the store? - #[must_use] - fn still_describes(&self, files: &FileStore) -> bool { - self.health == files.health_generation() - } - - /// Chunks re-hashed. - #[must_use] - pub fn checked(&self) -> u64 { - self.checked - } - - /// Chunks rewritten from the legacy copy. - #[must_use] - pub fn repaired(&self) -> u64 { - self.repaired +/// Flushed before any rename. On ext4 `auto_da_alloc` only orders the data before the +/// rename's own commit; it does not make the data durable, and btrfs has been observed +/// reordering. A name must never become visible on bytes that are not on the platter. +fn write_temp(temp_path: &Path, payload: &[u8]) -> Result<()> { + let mut f = OpenOptions::new() + .write(true) + .create_new(true) + .open(temp_path) + .map_err(|e| { + Error::Storage(format!( + "Failed to create temporary file {}: {e}", + temp_path.display() + )) + })?; + if let Err(e) = f.write_all(payload) { + let _ = std::fs::remove_file(temp_path); + return Err(Error::Storage(format!( + "Failed to write {}: {e}", + temp_path.display() + ))); } - - /// Chunks that could not be made good. - #[must_use] - pub fn unrepairable(&self) -> u64 { - self.unrepairable + if let Err(e) = f.sync_all() { + let _ = std::fs::remove_file(temp_path); + return Err(Error::Storage(format!( + "Failed to flush {}: {e}", + temp_path.display() + ))); } + Ok(()) } -/// Mark a retired environment directory as retired, from the inside, durably. +/// Write `payload` and publish it under `final_path`. /// -/// Called only after the directory has already been renamed aside, so it can never land -/// inside a live environment. See [`RETIRED_MARKER`] for why it goes inside. +/// The temp lives in the destination directory, so the publish is an intra-directory +/// rename: atomic on every filesystem we support, and needing only that one directory +/// Put `payload` on disk as `final_path`, durably. /// -/// # Errors -/// -/// Returns [`Error::Storage`] if it cannot be created or flushed. -fn mark_directory_retired(dir: &Path) -> std::result::Result<(), MarkFailure> { - match write_retirement_mark(dir) { - Ok(()) => Ok(()), - Err(e) if e.pre_existing => { - // Nothing here was created by this attempt, so there is nothing to take back. - // Removing a mark that was already there because re-flushing it failed is how - // a correctly retired directory comes to look unmarked, and an unmarked - // directory is restored as a live environment. - Err(MarkFailure { - pre_existing: true, - ..e - }) - } - Err(e) => { - // A half-written mark is worse than none: the caller puts the directory back - // under the live name and reopens it, and a mark left inside would have the - // next cleanup pass reap a live, open environment. If it cannot be taken away, - // say so, and the caller keeps the directory where nothing will open it. - let path = dir.join(RETIRED_MARKER); - match std::fs::remove_file(&path) { - Ok(()) => {} - Err(gone) if gone.kind() == std::io::ErrorKind::NotFound => {} - Err(stuck) => { - return Err(MarkFailure { - reason: format!( - "{e}. The partial mark at {} could not be removed either \ - ({stuck})", - path.display() - ), - mark_definitely_gone: false, - pre_existing: false, - }) - } - } - if let Err(flush) = crate::storage::file_store::fsync_path(dir) { - return Err(MarkFailure { - reason: format!( - "{e}. Removing the partial mark at {} could not be flushed \ - ({flush})", - path.display() - ), - mark_definitely_gone: false, - pre_existing: false, - }); - } - Err(MarkFailure { - reason: format!("{e}"), - mark_definitely_gone: true, - pre_existing: false, - }) - } - } -} - -/// Why a directory could not be marked retired, and whether it is safe to reopen. -#[derive(Debug)] -struct MarkFailure { - /// What went wrong, for the operator. - reason: String, - /// Is the directory provably free of a partial mark? - /// - /// Only then may the caller put it back under the live name. A mark left inside would - /// have the next cleanup pass reap a live, open environment. - mark_definitely_gone: bool, - /// Was the mark already there before this attempt? - /// - /// Then this attempt created nothing and must take nothing away. Removing a mark that - /// was already there because re-flushing it failed is how a correctly retired - /// directory comes to look unmarked, and an unmarked directory is restored as a live - /// environment. - pre_existing: bool, +/// Returns [`PutOutcome::Duplicate`] when the name is already taken. The name is a hash +/// of the content, so that is not treated as proof the bytes are right: the caller +/// re-reads and verifies them. +#[cfg(unix)] +fn publish( + temp_path: &Path, + final_path: &Path, + payload: &[u8], + shard: &Path, +) -> std::result::Result { + // On Unix nothing is ever created under the final name by a failing path: the bytes go + // to a temporary and only a successful rename gives them the real name. So every + // failure here leaves the name as it found it. + publish_via_rename(temp_path, final_path, payload, shard) + .map_err(PublishFailed::nothing_written) } -impl std::fmt::Display for MarkFailure { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.reason) - } +/// Put `payload` on disk as `final_path`, durably. See [`publish_in_place`] for why this +/// takes a different route off Unix. +#[cfg(not(unix))] +fn publish( + temp_path: &Path, + final_path: &Path, + payload: &[u8], + shard: &Path, +) -> std::result::Result { + let _ = temp_path; + let _ = shard; + publish_in_place(final_path, payload) } -/// Create the mark. See [`mark_directory_retired`], which owns the failure handling. -fn write_retirement_mark(dir: &Path) -> std::result::Result<(), MarkFailure> { - let path = dir.join(RETIRED_MARKER); - let mut file = match std::fs::OpenOptions::new() +/// Create the chunk under its final name and flush it. Everywhere but Unix. +/// +/// There is no way to flush a directory through the standard library, and Microsoft does +/// not document `MoveFileEx` as durable at return unless it is called with +/// `MOVEFILE_WRITE_THROUGH`, which std does not use. So off Unix a rename cannot be +/// relied on to have reached the disk before the legacy store is deleted. +/// +/// Creating the file under its final name sidesteps the rename entirely. Microsoft +/// documents that creation metadata is cached and that `FlushFileBuffers`, which +/// `sync_all` calls on Windows, is the way to flush it. So a successful create, write and +/// flush is a durable publication under a documented contract, with no directory flush +/// and no rename involved. +/// +/// The cost is that a crash mid-write leaves a partial file wearing a real chunk name. +/// That is why a duplicate re-reads and verifies rather than trusting the name, and why +/// the pre-retirement pass re-hashes everything before anything is deleted. +#[cfg(not(unix))] +fn publish_in_place( + final_path: &Path, + payload: &[u8], +) -> std::result::Result { + // Test-only, and here rather than after the write so that it means the same thing on + // both platforms: the file half of a dual write has not happened yet. On Unix the + // equivalent point is the temporary file written and the rename not yet made, which is + // also before the chunk's name exists on disk. Stopping after the write instead would + // put the file under its real name already, so a crash there is not between the two + // halves at all, and it could not demonstrate anything about the missing flush either: + // killing a process does not empty the page cache, so the bytes are still there to be + // read. Only losing power loses them, which no test that kills a process can stage. + #[cfg(any(test, feature = "test-utils"))] + halt_here_if_asked(HALT_BEFORE_PUBLISH, final_path); + let mut file = match OpenOptions::new() .write(true) .create_new(true) - .open(&path) + .open(final_path) { Ok(f) => f, - // Already there, from an attempt that got this far and no further. Flushed - // again rather than taken on trust: the attempt that wrote it may have been the - // one that could not flush it. - Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => { - // Something is already at that name. That it could not be created is not the - // same as its being a mark this node can read, and everything downstream - // deletes an environment on the strength of it. The rest of this file insists - // the name is not the evidence; this is the one place that was taking it. - if retirement_mark(dir) != RetirementMark::Present { - return Err(MarkFailure { - reason: format!( - "{} already exists but cannot be read as a retirement mark, so it \ - is not one this node will delete on. Check what is at that path.", - path.display() - ), - mark_definitely_gone: false, - pre_existing: true, - }); - } - return crate::storage::file_store::fsync_path(dir).map_err(|flush| MarkFailure { - reason: format!( - "{} is already there but could not be flushed: {flush}", - path.display() - ), - mark_definitely_gone: false, - pre_existing: true, - }); - } + // Someone got there first. Immutable content under a content-addressed name, so + // the caller verifies what is already there rather than assuming it is right. + Err(e) if e.kind() == ErrorKind::AlreadyExists => return Ok(PutOutcome::Duplicate), Err(e) => { - return Err(MarkFailure { - reason: format!("Could not mark {} as retired: {e}", path.display()), - mark_definitely_gone: true, - pre_existing: false, - }) + // Nothing was created, so nothing was spent. + return Err(PublishFailed::nothing_written(Error::Storage(format!( + "Failed to create chunk {}: {e}", + final_path.display() + )))); } }; - // For whoever reads the directory. To the node, presence is the whole signal. - if let Err(e) = file.write_all( - b"This chunk environment was verified as fully copied into the file store and \n\ -retired. It is being deleted; if it is still here, that was interrupted and the next \n\ -node start finishes it. Nothing needs it.\n", - ) { + if let Err(e) = file.write_all(payload) { drop(file); - return Err(MarkFailure { - reason: format!("Could not write {}: {e}", path.display()), - mark_definitely_gone: false, - pre_existing: false, + // Taken back if it can be. Whether it could is what the caller needs: the file + // was created by this call, so if it is still there the space is spent. + let left_behind = std::fs::remove_file(final_path).is_err(); + return Err(PublishFailed { + error: Error::Storage(format!("Failed to write {}: {e}", final_path.display())), + left_behind, }); } - file.sync_all().map_err(|e| MarkFailure { - reason: format!( - "Could not flush {}: {e}. Not deleting on the strength of a mark that may not \ - survive a power loss.", - path.display() - ), - mark_definitely_gone: false, - pre_existing: false, - })?; - // And the directory that now contains it. Flushing the file makes its contents - // durable; the entry naming it is in the directory, and on Unix that needs its own - // flush. Without this the mark can be missing after a crash from a directory that - // was in fact retired, which is the whole question this file answers. - crate::storage::file_store::fsync_path(dir).map_err(|e| MarkFailure { - reason: format!( - "Marked {} retired but could not flush {}: {e}. Not deleting on the strength \ - of a mark that may not survive a power loss.", - path.display(), - dir.display() - ), - mark_definitely_gone: false, - pre_existing: false, - }) -} - -/// Delete a retired directory in the background, without anything waiting for it. -/// -/// The caller is finished with it either way: the environment is closed, the directory is -/// under a name nothing looks for, and it carries its own mark, so an interrupted deletion -/// is finished by the next start. What matters is that neither shutdown nor startup ever -/// blocks on a recursive delete that can run for minutes. -fn delete_retired_directory(dir: PathBuf) { - // One at a time per directory. The driver asks for cleanup on every tick while - // anything is pending, and starting a fresh thread each time would leave hundreds of - // them asleep on the same path, all retrying the same failure. - if !REAPING.lock().insert(dir.clone()) { - return; - } - let named = dir.clone(); - let started = std::thread::Builder::new() - .name("chunk-store-retire".into()) - .spawn(move || { - let _done = ReapingGuard(dir.clone()); - for attempt in 1..=RETIRED_DELETE_ATTEMPTS { - match remove_marked_directory(&dir) { - Ok(()) => { - info!( - migration_event = "space_returned", - "Removed the retired chunk environment {} and returned its \ - space", - dir.display() - ); - return; - } - // Worth another go: on Windows a scanner or an antivirus can hold a - // handle inside it for a moment, and a partial delete leaves less to - // do next time. - Err(e) if attempt < RETIRED_DELETE_ATTEMPTS => { - debug!( - "Could not delete {} (attempt {attempt}): {e}. Trying again.", - dir.display() - ); - std::thread::sleep( - (RETIRED_DELETE_BACKOFF * attempt).min(RETIRED_DELETE_BACKOFF_MAX), - ); - } - Err(e) => warn!( - "The chunk environment has been retired but {} could not be \ - deleted: {e}. Its space is not returned until it is, and the node \ - needs nothing from it. The next start tries again.", - dir.display() - ), - } - } + if let Err(e) = file.sync_all() { + drop(file); + // Taken back if it can be. Whether it could is what the caller needs: the file + // was created by this call, so if it is still there the space is spent. + let left_behind = std::fs::remove_file(final_path).is_err(); + return Err(PublishFailed { + error: Error::Storage(format!("Failed to flush {}: {e}", final_path.display())), + left_behind, }); - if let Err(e) = started { - REAPING.lock().remove(&named); - warn!( - "Could not start the thread to delete the retired chunk environment {}: {e}. \ - The next start sweeps it.", - named.display() - ); - } -} - -/// Directories a reaper thread is already working on. -static REAPING: parking_lot::Mutex> = parking_lot::Mutex::new(BTreeSet::new()); - -/// Releases a directory from [`REAPING`] however its thread ends. -struct ReapingGuard(PathBuf); - -impl Drop for ReapingGuard { - fn drop(&mut self) { - REAPING.lock().remove(&self.0); - } -} - -/// Delete a retired directory, taking its mark away last of all. -/// -/// `remove_dir_all` walks in whatever order the filesystem hands back, so it can unlink -/// the mark and then fail on the next entry, which is exactly what a Windows sharing -/// violation on the data file produces. What is left is a genuinely retired, partly -/// deleted directory carrying no evidence that it was retired, and the next start would -/// read that as an intact environment and restore it. -/// -/// Emptying it first and removing the mark last means the mark is only ever absent from a -/// directory that has nothing else left in it. -/// -/// # Errors -/// -/// Returns the underlying I/O error. The directory is left with its mark intact on every -/// failure that happens before the mark is reached. -fn remove_marked_directory(dir: &Path) -> std::io::Result<()> { - // Never through a link. An operator who points the chunk environment at another - // volume leaves a symlink here, and walking it would delete the contents of a - // directory that is not this node's to delete. Retirement refuses such a root before - // it gets this far; this is the second line, because the check and the walk are not - // one operation. - if std::fs::symlink_metadata(dir)?.file_type().is_symlink() { - return Err(std::io::Error::other(format!( - "{} is a link, not a directory. Refusing to delete through it.", - dir.display() - ))); - } - for entry in std::fs::read_dir(dir)? { - let entry = entry?; - if entry.file_name() == RETIRED_MARKER { - continue; - } - let path = entry.path(); - if entry.file_type()?.is_dir() { - std::fs::remove_dir_all(&path)?; - } else { - std::fs::remove_file(&path)?; - } - } - match std::fs::remove_file(dir.join(RETIRED_MARKER)) { - Ok(()) => {} - Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} - Err(e) => return Err(e), - } - match std::fs::remove_dir(dir) { - Ok(()) => Ok(()), - Err(e) => { - // The mark is gone and the directory is not, which is the one state the whole - // scheme says cannot happen: a start that found it would read an unmarked - // directory as an intact environment. Put the mark back before giving up. - if let Err(remark) = mark_directory_retired(dir) { - error!( - "Could not remove {} ({e}) and could not restore its retirement mark \ - ({remark}). It is empty and nothing needs it; delete it by hand.", - dir.display() - ); - } - Err(e) - } - } -} - -/// What a directory's own contents say about whether it was retired. -/// -/// Three answers, not two. Reading the mark can fail for reasons that are neither yes nor -/// no: a permission change, a descriptor limit, a filesystem that has gone away underneath -/// the node. Folding that into "no" is the failure mode the rest of this file exists to -/// avoid, and it fails in the worst direction: a retired environment that reads as unmarked -/// is put back under the live name and reopened, and its keys re-enter a commitment they -/// have already left. -/// -/// So an unreadable answer is its own answer, and the two questions callers actually ask -/// are asked separately. Neither of them treats "cannot tell" as permission. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum RetirementMark { - /// The directory carries its mark. It is the remains of a removal. - Present, - /// The directory carries no mark, and that is known rather than assumed. - Absent, - /// Whether it carries one could not be determined. - Unknown, -} - -impl RetirementMark { - /// May this directory be deleted, or treated as already gone? - /// - /// Only a mark actually read says yes. Deleting on a guess destroys chunks. - const fn permits_removal(self) -> bool { - matches!(self, Self::Present) - } - - /// May this directory be opened and served from? - /// - /// Only a mark known to be absent says yes. Opening a retired environment puts keys - /// back into a commitment they have already left. - const fn permits_opening(self) -> bool { - matches!(self, Self::Absent) } + Ok(PutOutcome::New) } -/// Has this directory been retired? +/// Write a temp beside the target and rename it into place. Unix only. /// -/// A link is never treated as retired, whatever it points at: the mark would have been -/// written through it into somebody else's directory, and acting on it would delete -/// somebody else's data. A path whose kind cannot be determined is not a link either way, -/// and is reported as unknown rather than as a link, so that neither question gets a yes. -fn retirement_mark(dir: &Path) -> RetirementMark { - match std::fs::symlink_metadata(dir) { - Ok(meta) if meta.file_type().is_symlink() => return RetirementMark::Absent, - Ok(_) => {} - // Nothing here at all, which is the ordinary case on a node that has already - // finished or never had a legacy store. There is no mark because there is nothing - // to carry one, and that is known rather than undetermined. - Err(e) if e.kind() == std::io::ErrorKind::NotFound => return RetirementMark::Absent, - Err(e) => { - // Debug, not warn: this is asked on every tick, so a warn here would be a - // wall of the same line. The operator-facing version is the retirement - // blocker, which says what it means for the node. - debug!( - "Could not tell what {} is ({e}); treating it as neither removable nor \ - openable until it can be read", - dir.display() - ); - return RetirementMark::Unknown; - } - } - match dir.join(RETIRED_MARKER).try_exists() { - Ok(true) => RetirementMark::Present, - Ok(false) => RetirementMark::Absent, - Err(e) => { - debug!( - "Could not read the retirement mark in {} ({e}); treating it as neither \ - removable nor openable until it can be read", - dir.display() - ); - RetirementMark::Unknown - } - } -} - -/// Is this path a symbolic link, or something whose kind cannot be determined? -/// -/// Unknown counts as yes. Every caller is deciding whether it is safe to delete through -/// the path, and a question that cannot be answered is not a yes to that. -fn is_a_link(path: &Path) -> bool { - std::fs::symlink_metadata(path).map_or(true, |m| m.file_type().is_symlink()) -} - -/// Finish a removal a previous run did not, before anything tries to open the environment. -/// -/// The only thing that counts as evidence is the directory's own mark. An open that fails -/// is not: `open_legacy` queries free space, maps the file, takes a write transaction and -/// scans every key, so a full disk, a permission change, a mapping limit or a transient -/// I/O fault all look identical to corruption, and deleting on any of those would destroy -/// a perfectly good environment. -fn finish_interrupted_retirement(root_dir: &Path) -> LiveEnvironment { - let env = root_dir.join(LEGACY_ENV_DIR); - // Three answers, three branches. Asking only whether it may be removed and letting - // everything else fall through would put "cannot tell" back on the opening path, which - // is the whole failure this is three states to avoid: the mark check can fail for a - // moment and succeed the next, and the open in between would resurrect a store that - // really had been retired. +/// Places the bytes and nothing more. Making the name durable is +/// [`flush_publication`]'s job, kept separate so a caller can tell a publish that spent no +/// space from one that spent it and could not be reported. +#[cfg(unix)] +fn publish_via_rename( + temp_path: &Path, + final_path: &Path, + payload: &[u8], + _shard: &Path, +) -> Result { + // Content is immutable and the name is its hash, so an existing file already holds + // exactly these bytes. Skipping the write is both cheaper and safer than replacing + // it: on Windows a rename over a file another thread has open fails outright. // - // Asked of the mark alone, with no separate "is it there" first. A `try_exists` that - // could not answer would have folded straight back into "nothing here" and skipped both - // branches below, which is the same fold one level up. The mark already tells the three - // apart: a path that is not there carries no mark and says so, and a path that cannot be - // reached at all says it cannot be reached. - // Asked once. Asking twice is asking two different questions: the answer can change - // between them, and a second answer of "cannot tell" after a first of "retired" fell - // through to opening the very directory the first answer said not to open. - let mark = retirement_mark(&env); - if mark == RetirementMark::Unknown { - error!( - "{} is under the live name and this node cannot tell whether it was retired. \ - It will NOT be opened and it will NOT be removed. The node serves from files \ - alone. Check that the directory and anything inside it can be read.", - env.display() - ); - return LiveEnvironment::None; - } - if mark.permits_removal() { - // Its own contents say it was retired, so whatever name it is wearing now, it is - // the remains of a removal that a power loss undid the rename of. - warn!( - "{} carries its own retirement mark, so it is what an interrupted removal left \ - behind rather than a live environment. Finishing that removal.", - env.display() - ); - // Renamed rather than deleted here, so the node can get on with starting: the - // deletion itself is detached below and can take minutes on a large store. Under a - // name nothing else is using, so a tombstone whose deletion is still running does - // not force a synchronous delete first. - let tombstone = free_tombstone_path(root_dir); - if let Err(e) = std::fs::rename(&env, &tombstone) { - error!( - "{} carries its own retirement mark but could not be moved aside: {e}. It \ - will NOT be opened: it says it has been retired, so it may be partly \ - deleted, and its chunks are in the file store. The node serves from files \ - alone and the next start tries again.", - env.display() - ); - sweep_retired_legacy(root_dir); - return LiveEnvironment::None; - } - } - sweep_retired_legacy(root_dir); - LiveEnvironment::WhateverIsOnDisk -} - -/// Whether the ordinary open may look at what is under the live environment name. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum LiveEnvironment { - /// Nothing is claiming it should not be opened. - WhateverIsOnDisk, - /// A directory under the live name says it has been retired, and could not be moved - /// out of the way. It must not be opened: a retired directory may be partly deleted, - /// and opening it would put its keys back into a commitment they have left. - None, -} - -fn sweep_retired_legacy(root_dir: &Path) { - let tombstones = retired_tombstones(root_dir); - if tombstones.is_empty() { - return; - } - // Flushed first, and only best effort is not good enough here for the same reason it - // was not good enough when the rename was made: deleting the contents of a directory - // whose new name may not have reached the disk is what turns a power loss into a - // resurrected, half-empty environment. If it cannot be flushed, leave them for a later - // start. It costs disk, not data. - if let Err(e) = crate::storage::file_store::fsync_path(root_dir) { - warn!( - "Leaving {} retired chunk environment(s) in place: {} could not be flushed \ - ({e}), so the rename that put them there may not be on disk yet.", - tombstones.len(), - root_dir.display() - ); - return; - } - for tombstone in tombstones { - // The name is not the evidence. Only the directory's own mark is: a crash between - // the rename and the mark leaves an intact environment sitting under the retired - // name, and deleting that because of what it is called would destroy every chunk - // in it. - let mark = retirement_mark(&tombstone); - if mark == RetirementMark::Unknown { - // Neither restored nor deleted. Restoring would put a directory that may be - // half-deleted back under the live name for the next start to open, and - // deleting would destroy an intact one. It costs disk until somebody looks, - // which is the right price for not knowing. - warn!( - "{} cannot be classified: this node cannot tell whether it carries a \ - retirement mark, so it will be neither restored nor deleted. Check that \ - the directory and anything inside it can be read.", - tombstone.display() - ); - continue; - } - if mark.permits_removal() { - // The mark is re-established before anything is deleted on the strength of - // it. A retirement that failed part-way can leave one that was never flushed, - // and this is the pass that would otherwise act on it thirty seconds after - // the failure that said it would be left alone. - if let Err(e) = mark_directory_retired(&tombstone) { - warn!( - "{} says it was retired but that could not be confirmed ({e}). \ - Leaving it.", - tombstone.display() - ); - continue; - } - // Detached, so a node starting beside a large leftover directory serves - // immediately rather than waiting out a recursive delete before it opens its - // store. - delete_retired_directory(tombstone); - continue; - } - restore_unmarked_environment(root_dir, &tombstone); - } -} - -/// Put an intact environment back under its own name. -/// -/// An environment under the retired name with no mark inside it was renamed and then -/// interrupted before it could be marked. Nothing was deleted, so it is whole, and the -/// answer is to give it its name back and let the migration run again from the beginning: -/// every gate is re-derived, and a second retirement costs a pass, not data. -fn restore_unmarked_environment(root_dir: &Path, tombstone: &Path) { - // An empty one is what a deletion that removed the contents and the mark and then - // could not remove the directory leaves. There is nothing in it to restore, and - // putting it back under the live name would strand an empty path the node then tries - // to open. - if std::fs::read_dir(tombstone).is_ok_and(|mut entries| entries.next().is_none()) { - if let Err(e) = std::fs::remove_dir(tombstone) { - warn!("Could not remove the empty {}: {e}", tombstone.display()); - } - return; - } - let env = root_dir.join(LEGACY_ENV_DIR); - if env.try_exists().unwrap_or(true) { - // Both names are taken, so which one the node should serve is not this code's - // decision to make. - error!( - "{} and {} both exist, and {} carries no retirement mark, so it may hold \ - chunks. Neither has been touched. Move or remove one by hand: the node is \ - using {}.", - env.display(), - tombstone.display(), - tombstone.display(), - env.display() - ); - return; - } - match std::fs::rename(tombstone, &env) { - Ok(()) => { - let _ = crate::storage::file_store::fsync_path(root_dir); - warn!( - "{} was moved aside for retirement but never marked retired, so it is \ - intact. It has been restored to {} and the migration starts again.", - tombstone.display(), - env.display() - ); - } - Err(e) => error!( - "{} carries no retirement mark, so it may hold chunks, but it could not be \ - restored to {}: {e}. It has not been deleted.", - tombstone.display(), - env.display() - ), - } -} - -/// Every retired environment directory under `root_dir`. -/// -/// More than one can be there: a node that retires, is restarted before the deletion -/// finishes, and somehow acquires another environment would leave the first behind. Each -/// is named so it cannot collide with the next. -fn retired_tombstones(root_dir: &Path) -> Vec { - let prefix = format!("{LEGACY_ENV_DIR}{RETIRED_SUFFIX}"); - let entries = match std::fs::read_dir(root_dir) { - Ok(entries) => entries, - Err(e) => { - // Cannot tell. Not the same as nothing here, and the caller uses this to - // decide whether cleanup is finished, so answer with the one that keeps it - // looking rather than the one that declares victory. - warn!( - "Could not list {} to look for retired chunk environments: {e}", - root_dir.display() - ); - return vec![root_dir.join(&prefix)]; - } - }; - let mut found = Vec::new(); - for entry in entries { - match entry { - Ok(entry) => { - if entry - .file_name() - .to_str() - .is_some_and(|n| n.starts_with(&prefix)) - { - found.push(entry.path()); - } - } - // One unreadable entry is not evidence there is nothing here, and the caller - // uses this to decide whether cleanup is finished. Answer with the one that - // keeps it looking. - Err(e) => { - warn!( - "Could not read an entry of {} while looking for retired chunk \ - environments: {e}", - root_dir.display() - ); - found.push(root_dir.join(&prefix)); - } - } - } - found -} - -/// A directory name to retire the environment under that nothing else is using. -/// -/// A fixed name would collide with a tombstone whose deletion is still running, and -/// clearing that one first would put a synchronous recursive delete back on the path this -/// is trying to keep clear. -fn free_tombstone_path(root_dir: &Path) -> PathBuf { - let base = root_dir.join(format!("{LEGACY_ENV_DIR}{RETIRED_SUFFIX}")); - if !base.try_exists().unwrap_or(true) { - return base; - } - for n in 1..=MAX_TOMBSTONES { - let candidate = root_dir.join(format!("{LEGACY_ENV_DIR}{RETIRED_SUFFIX}.{n}")); - if !candidate.try_exists().unwrap_or(true) { - return candidate; - } - } - // Every name taken, which means many retirements have been interrupted without their - // deletions finishing. Reuse the base: the rename fails, retirement defers, and the - // operator sees a directory full of them. - base -} - -/// Whether a legacy environment is on disk under `root_dir`. -/// -/// # Errors -/// -/// Returns [`Error::Storage`] if the answer cannot be determined. `Path::exists` would -/// turn a permission problem into "absent", and a node that starts in file-only mode -/// beside a `chunks.mdb` holding every chunk it has stops serving all of them. -pub fn legacy_present(root_dir: &Path) -> Result { - let path = root_dir.join(LEGACY_ENV_DIR).join(LEGACY_DATA_FILE); - path.try_exists().map_err(|e| { - Error::Storage(format!( - "Cannot tell whether the legacy chunk environment {} exists: {e}. Refusing to \ - start rather than ignore it.", - path.display() - )) - }) -} - -/// How long to sleep after copying `bytes` to hold the copier to a rate ceiling. -fn throttle_delay(bytes: u64, mib_per_sec: u64) -> Option { - if mib_per_sec == 0 { - return None; - } - let per_sec = mib_per_sec.saturating_mul(1024 * 1024); - if per_sec == 0 { - return None; - } - let micros = bytes.saturating_mul(1_000_000) / per_sec; - if micros == 0 { - None + // The caller flushes either way. A name that is already there is not proof it is + // durable: + // the write that put it there may have been this store's own previous attempt, whose + // rename landed and whose directory flush then failed. That attempt returned an + // error, so nothing was retired on the strength of it, but if this call reported a + // durable duplicate without flushing, the retry would silently launder an unflushed + // rename into a copy that authorises deleting the last other one. + let outcome = if final_path.exists() { + PutOutcome::Duplicate } else { - Some(Duration::from_micros(micros)) - } -} - -/// Merge two ascending key sequences into one, dropping duplicates. -fn merge_sorted<'a, I>(sorted: &[XorName], other: I) -> Vec -where - I: Iterator, -{ - let other: Vec = other.copied().collect(); - let mut out = Vec::with_capacity(sorted.len() + other.len()); - let mut a = sorted.iter().copied().peekable(); - let mut b = other.into_iter().peekable(); - loop { - match (a.peek(), b.peek()) { - (Some(x), Some(y)) => match x.cmp(y) { - std::cmp::Ordering::Less => out.extend(a.next()), - std::cmp::Ordering::Greater => out.extend(b.next()), - std::cmp::Ordering::Equal => { - out.extend(a.next()); - let _ = b.next(); - } - }, - (Some(_), None) => out.extend(a.next()), - (None, Some(_)) => out.extend(b.next()), - (None, None) => break, - } - } - out -} - -#[cfg(test)] -#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] -mod tests { - use super::*; - use crate::storage::migration::{now_unix, rank_closest_first, MIN_RETIRE_DELAY_HOURS}; - use tempfile::TempDir; - - /// Everything currently legacy-only, as the set a test has "approved" for shedding. - fn approved_shed(store: &ChunkStore) -> BTreeSet { - store.legacy_only_keys().into_iter().collect() - } - - /// A token that is never cancelled, for tests that are not exercising shutdown. - fn never_cancelled() -> CancellationToken { - CancellationToken::new() - } - - /// Put a store through every gate a real node passes before it may retire. - /// - /// Deliberately not a shortcut around them: `retire_legacy` rechecks the whole set - /// itself, so a test that skipped them would exercise a path production never takes. - fn open_the_retirement_gate(store: &ChunkStore) { - store.commit_to_files().expect("commit"); - store.note_commitment_rebuilt(); - store.note_commitment_rebuilt(); - store.force_migration_state(|s| { - s.committed_at_unix = - Some(now_unix().saturating_sub(MIN_RETIRE_DELAY_HOURS * 3600 + 60)); - }); - } - - /// Content plus the address it hashes to. - fn addressed(seed: &str) -> (XorName, Vec) { - let content = format!("chunk-content-{seed}").into_bytes(); - (crate::client::compute_address(&content), content) - } - - fn test_config(dir: &TempDir) -> ChunkStoreConfig { - ChunkStoreConfig { - root_dir: dir.path().to_path_buf(), - ..ChunkStoreConfig::test_default() - } - } - - async fn open(dir: &TempDir) -> ChunkStore { - ChunkStore::new(test_config(dir)).await.expect("open store") - } - - /// Populate a legacy LMDB environment the way an existing node would have one, then - /// close it so the facade can adopt it. - async fn seed_legacy(dir: &TempDir, seeds: &[&str]) -> Vec { - let lmdb = LmdbStorage::new(LmdbStorageConfig { - root_dir: dir.path().to_path_buf(), - verify_on_read: true, - max_map_size: 0, - disk_reserve: 0, - }) - .await - .expect("open legacy"); - let mut keys = Vec::new(); - for seed in seeds { - let (addr, content) = addressed(seed); - lmdb.put(&addr, &content).await.expect("legacy put"); - keys.push(addr); - } - lmdb.wait_idle().await; - drop(lmdb); - keys - } - - #[tokio::test] - async fn a_fresh_node_never_creates_a_legacy_environment() { - let dir = TempDir::new().expect("temp dir"); - let store = open(&dir).await; - - assert!(!store.has_legacy()); - assert_eq!(store.migration_phase(), MigrationPhase::FilesOnly); - assert!(!dir.path().join(LEGACY_ENV_DIR).exists()); - - let (addr, content) = addressed("fresh"); - assert!(store.put(&addr, &content).await.expect("put")); - assert_eq!( - store.get(&addr).await.expect("get").expect("present"), - content - ); - } - - #[tokio::test] - async fn an_existing_legacy_store_is_adopted_and_served() { - let dir = TempDir::new().expect("temp dir"); - let keys = seed_legacy(&dir, &["a", "b", "c"]).await; - let store = open(&dir).await; - - assert!(store.has_legacy()); - assert_eq!(store.migration_phase(), MigrationPhase::Bridging); - assert_eq!(store.current_chunks().expect("count"), 3); - for key in &keys { - assert!(store.exists(key).expect("exists"), "union must see it"); - assert!(store.get(key).await.expect("get").is_some()); - } - } - - #[tokio::test] - async fn the_union_key_set_is_sorted_and_free_of_duplicates() { - let dir = TempDir::new().expect("temp dir"); - let mut expected = seed_legacy(&dir, &["u1", "u2", "u3", "u4"]).await; - let store = open(&dir).await; - - // One chunk written now lives in both backings, and must be counted once. - let (addr, content) = addressed("u2"); - assert!(expected.contains(&addr)); - assert!(!store.put(&addr, &content).await.expect("put")); - - let (fresh, fresh_content) = addressed("u5"); - store.put(&fresh, &fresh_content).await.expect("put"); - expected.push(fresh); - expected.sort_unstable(); - - let keys = store.all_keys().await.expect("all_keys"); - assert_eq!(keys, expected); - assert_eq!(store.current_chunks().expect("count"), 5); - } - - /// The legacy environment takes no new disk during the bridge. - /// - /// Both stores sit on one disk, each measures the same free space, and neither knows - /// what the other is about to spend. A chunk written to both could be admitted twice - /// against one lot of headroom, and enough of them could cross the reserve together - /// and fill the volume this migration exists to free. - /// - /// So the environment is pinned to what it already occupies. It still takes the - /// rollback copy when it has room of its own, which on a real node it usually does: - /// this migration exists because deleting millions of chunks left the free list full - /// and returned nothing to the filesystem. What it will not do is grow. - #[tokio::test] - async fn the_legacy_environment_takes_no_new_disk_during_the_bridge() { - let dir = TempDir::new().expect("temp dir"); - seed_legacy(&dir, &["seed"]).await; - let store = open(&dir).await; - - let data_file = dir.path().join(LEGACY_ENV_DIR).join(LEGACY_DATA_FILE); - let before = std::fs::metadata(&data_file).expect("meta").len(); - - for seed in ["dual-1", "dual-2", "dual-3", "dual-4"] { - let (addr, content) = addressed(seed); - assert!(store.put(&addr, &content).await.expect("put")); - assert!( - store.exists(&addr).expect("exists"), - "the file store is the one that has to have it" - ); - } - store.wait_idle().await; - - let after = std::fs::metadata(&data_file).expect("meta").len(); - assert_eq!( - after, before, - "the environment must not claim disk the file store is also counting on" - ); - } - - #[tokio::test] - async fn the_copier_moves_keys_into_files_and_is_resumable() { - let dir = TempDir::new().expect("temp dir"); - let keys = seed_legacy(&dir, &["c1", "c2", "c3", "c4", "c5"]).await; - let store = open(&dir).await; - assert_eq!(store.legacy_only_keys().len(), 5); - - let first = store - .copy_batch(&keys[..2], 0, 0, &never_cancelled()) - .await - .expect("copy first batch"); - assert_eq!(first.copied, 2); - assert_eq!(store.legacy_only_keys().len(), 3); - store.wait_idle().await; - drop(store); - - // A restart re-derives what is left from the filesystem: no progress file to - // corrupt, and no work repeated. - let store = open(&dir).await; - assert_eq!(store.legacy_only_keys().len(), 3); - let rest = store - .copy_batch(&keys, 0, 0, &never_cancelled()) - .await - .expect("copy rest"); - assert_eq!(rest.copied, 3); - assert!(store.legacy_only_keys().is_empty()); - assert_eq!(store.current_chunks().expect("count"), 5); - } - - #[tokio::test] - async fn a_delete_reaches_both_stores() { - let dir = TempDir::new().expect("temp dir"); - let keys = seed_legacy(&dir, &["d1", "d2"]).await; - let store = open(&dir).await; - - let target = keys.first().copied().expect("a key"); - assert!(store.delete(&target).await.expect("delete")); - assert!(!store.exists(&target).expect("exists")); - assert!(store.get(&target).await.expect("get").is_none()); - assert_eq!(store.current_chunks().expect("count"), 1); - - // And it stays gone across a restart, which is what proves it left the legacy - // environment too rather than only the union view. - store.wait_idle().await; - drop(store); - let store = open(&dir).await; - assert!(!store.exists(&target).expect("exists")); - } - - #[tokio::test] - async fn the_copier_does_not_resurrect_a_deleted_chunk() { - let dir = TempDir::new().expect("temp dir"); - let keys = seed_legacy(&dir, &["r1", "r2"]).await; - let store = open(&dir).await; - - let target = keys.first().copied().expect("a key"); - store.delete(&target).await.expect("delete"); - - let report = store - .copy_batch(&keys, 0, 0, &never_cancelled()) - .await - .expect("copy"); - assert_eq!(report.copied, 1, "only the surviving chunk may be copied"); - assert!(!store.exists(&target).expect("exists")); - } - - /// A store left partway through by the release that shipped is picked up, not restarted. - /// - /// This release lands on nodes that are already migrating under the one before it, so what - /// matters is not what it does to a fresh store but what it does to a half-finished one. - /// It must re-read that state and change nothing: the phase, the first-start time the - /// waves are measured from, what the node committed to giving up, and the keys still only - /// in the legacy store all have to survive, or a node restarts a clock it had nearly run - /// down, or re-copies what it already copied. - /// - /// Nothing in this release writes migration state, and this is what says so. - #[tokio::test] - async fn a_store_left_midway_by_the_previous_release_keeps_its_place() { - let dir = TempDir::new().expect("temp dir"); - let keys = seed_legacy(&dir, &["k1", "k2", "k3"]).await; - - let started_at = { - let store = open(&dir).await; - // Partway: one copied, two still only in the legacy store, which is where a node - // that ran out of disk under the previous release sits. - store - .copy_batch(&keys[..1], 0, 0, &never_cancelled()) - .await - .expect("copy"); - store.commit_to_files().expect("commit"); - assert_eq!(store.migration_phase(), MigrationPhase::Committed); - assert_eq!(store.legacy_only_keys().len(), 2); - store.migration_state().first_start_unix - }; - - // The upgrade: a new process opening the same directory. - let store = open(&dir).await; - assert_eq!( - store.migration_phase(), - MigrationPhase::Committed, - "an upgrade must not put a committed node back to bridging" - ); - assert_eq!( - store.migration_state().first_start_unix, - started_at, - "the waves are measured from this, so restarting it restarts the schedule" - ); - assert_eq!( - store.migration_state().shed_key_count, - 2, - "what the node committed to giving up must survive the upgrade" - ); - assert_eq!( - store.legacy_only_keys().len(), - 2, - "the keys still to copy must not be recounted from scratch" - ); - - // And everything is still servable, out of whichever store holds it. - for key in &keys { - assert!( - store.get(key).await.expect("get").is_some(), - "a chunk stopped being servable across the upgrade" - ); - } - } - - #[tokio::test] - async fn committing_narrows_the_commitment_but_not_what_is_served() { - let dir = TempDir::new().expect("temp dir"); - let keys = seed_legacy(&dir, &["k1", "k2", "k3"]).await; - let store = open(&dir).await; - - // Copy one, leave two behind as if the disk had run out. - store - .copy_batch(&keys[..1], 0, 0, &never_cancelled()) - .await - .expect("copy"); - - // While bridging, the node still claims everything it can serve. - assert_eq!( - store.committable_keys().await.expect("committable").len(), - 3 - ); - - store.commit_to_files().expect("commit"); - assert_eq!(store.migration_phase(), MigrationPhase::Committed); - assert_eq!(store.migration_state().shed_key_count, 2); - - // It now claims only what it will keep... - assert_eq!( - store.committable_keys().await.expect("committable").len(), - 1 - ); - // ...while still serving everything it ever claimed. - assert_eq!(store.all_keys().await.expect("all_keys").len(), 3); - for key in &keys { - assert!(store.get(key).await.expect("get").is_some()); - } - } - - #[tokio::test] - async fn retirement_is_refused_until_every_gate_is_satisfied() { - let dir = TempDir::new().expect("temp dir"); - let keys = seed_legacy(&dir, &["g1", "g2"]).await; - let mut config = test_config(&dir); - config.migration.retire_legacy = true; - let store = ChunkStore::new(config).await.expect("open"); - - // Still bridging. - assert!(store - .retirement_blocker(|_| false) - .expect("blocked") - .contains("Bridging")); - - store - .copy_batch(&keys, 0, 0, &never_cancelled()) - .await - .expect("copy"); - store.commit_to_files().expect("commit"); - - // No commitment rebuild observed yet. - assert!(store - .retirement_blocker(|_| false) - .expect("blocked") - .contains("commitment rebuilds")); - - store.note_commitment_rebuilt(); - store.note_commitment_rebuilt(); - - // The retention delay has not elapsed. - assert!(store - .retirement_blocker(|_| false) - .expect("blocked") - .contains("retirement delay")); - - store.force_migration_state(|s| { - s.committed_at_unix = - Some(now_unix().saturating_sub(MIN_RETIRE_DELAY_HOURS * 3600 + 60)); - }); - assert!(store.retirement_blocker(|_| false).is_none()); - } - - #[tokio::test] - async fn a_chunk_still_answerable_vetoes_retirement() { - let dir = TempDir::new().expect("temp dir"); - seed_legacy(&dir, &["h1", "h2"]).await; - let mut config = test_config(&dir); - config.migration.retire_legacy = true; - let store = ChunkStore::new(config).await.expect("open"); - - // Shed both, as a node short of disk would. - store.commit_to_files().expect("commit"); - store.note_commitment_rebuilt(); - store.note_commitment_rebuilt(); - store.force_migration_state(|s| { - s.committed_at_unix = - Some(now_unix().saturating_sub(MIN_RETIRE_DELAY_HOURS * 3600 + 60)); - }); - - // The pruner's existing retention contract, reused verbatim: a key the node - // could still be challenged on keeps its last local copy. - assert!(store - .retirement_blocker(|_| true) - .expect("blocked") - .contains("still answerable")); - assert!(store.retirement_blocker(|_| false).is_none()); - } - - /// The stock configuration retires, with no environment variable and no operator step. - /// - /// This is the property the whole release rests on. Deleting `chunks.mdb` is the only - /// step that returns disk: LMDB never gives freed pages back, which is why the fleet - /// deleted millions of chunks and recovered nothing. A build that shipped with this - /// off would migrate every node and reclaim not one byte. - #[tokio::test] - async fn the_shipped_configuration_retires_without_an_operator_setting_anything() { - // Asked of the configuration a node actually builds, not of the constant behind - // it, so neither the constant nor a serde default nor the `Default` impl can turn - // retirement off without this failing. - assert!( - crate::storage::MigrationConfig::default().retire_legacy, - "this release must delete the legacy environment, or it frees no disk" - ); - - let dir = TempDir::new().expect("temp dir"); - let keys = seed_legacy(&dir, &["shipped"]).await; - let store = open(&dir).await; - store - .copy_batch(&keys, 0, 0, &never_cancelled()) - .await - .expect("copy"); - open_the_retirement_gate(&store); - assert!( - store.retirement_blocker(|_| false).is_none(), - "with every gate met, the shipped configuration must not refuse to retire" - ); - } - - /// Retirement waits for a read that is already running. - /// - /// The window this closes: a verifying read finds rotted bytes, throws the file away, - /// and has not yet reached the legacy copy that would replace it. If retirement ran in - /// that gap it would delete the only remaining copy. Holding the barrier shared for - /// the whole read, and exclusively for the removal, is what makes that impossible. - #[tokio::test] - async fn retirement_waits_for_a_read_that_is_already_running() { - let dir = TempDir::new().expect("temp dir"); - let keys = seed_legacy(&dir, &["held"]).await; - let store = Arc::new(open(&dir).await); - store - .copy_batch(&keys, 0, 0, &never_cancelled()) - .await - .expect("copy"); - open_the_retirement_gate(&store); - let proof = store - .verify_before_retire(0, &never_cancelled()) - .await - .expect("verify"); - - // Stand in for a read that has started and not finished. - let reading = store.retirement.read().await; - - let retiring = { - let store = Arc::clone(&store); - let approved = approved_shed(&store); - tokio::spawn(async move { - store - .retire_legacy(&proof, &|_: &XorName| false, &approved) - .await - }) - }; - - // It must not have got anywhere. Given a generous window rather than a tight one, - // so this fails on the behaviour rather than on scheduling luck. - tokio::time::sleep(Duration::from_millis(200)).await; - assert!( - !retiring.is_finished(), - "retirement removed the legacy environment while a read was still running" - ); - assert!( - store.has_legacy(), - "the legacy environment went while a read was still running" - ); - - drop(reading); - let freed = retiring.await.expect("join").expect("retire"); - assert!(freed > 0, "retirement should have freed the environment"); - assert!(!store.has_legacy()); - } - - /// A good copy is never turned away because a damaged one wears its name. - /// - /// Two shapes of damage, because they are caught differently: a short file, which is - /// what an interrupted create leaves on a platform that writes under the final name, - /// and a full-length file with wrong bytes, which is what rot leaves. Answering - /// "already have it" to either discards the copy that would fix it, and nothing offers - /// it again. - #[tokio::test] - async fn a_damaged_chunk_is_repaired_from_the_copy_being_offered() { - let dir = TempDir::new().expect("temp dir"); - let store = open(&dir).await; - let (addr, content) = addressed("repairable-by-offer"); - store.put(&addr, &content).await.expect("put"); - - // Intact: the offer is correctly refused. - assert!(store.holds_verified(&addr, &content).await); - - // Truncated. - let path = dir - .path() - .join("chunks") - .join(format!("{:02x}", addr.last().copied().unwrap_or(0))) - .join(hex::encode(addr)); - std::fs::write(&path, &content[..content.len() / 2]).expect("truncate"); - assert!( - store.holds_verified(&addr, &content).await, - "a short file must be replaced from the offered copy, not left in place" - ); - assert_eq!( - store.get(&addr).await.expect("get").expect("present"), - content - ); - - // Same length, wrong bytes. - let rotted = vec![b'x'; content.len()]; - assert_ne!(rotted, content); - std::fs::write(&path, &rotted).expect("rot"); - assert!( - store.holds_verified(&addr, &content).await, - "a rotted file must be replaced from the offered copy" - ); - assert_eq!( - store.get(&addr).await.expect("get").expect("present"), - content - ); - } - - /// A chunk held only in the legacy environment is checked, not assumed good. - /// - /// The bytes in there can be wrong too, and when the copier finds that out it drops - /// the key from the union view. Having turned the good copy away on the strength of - /// the key being present, the node would then hold nothing at all. - #[tokio::test] - async fn a_legacy_only_chunk_with_wrong_bytes_is_replaced_by_the_offer() { - let dir = TempDir::new().expect("temp dir"); - let keys = seed_legacy(&dir, &["legacy-side"]).await; - let key = *keys.first().expect("one key"); - let content = format!("chunk-content-{}", "legacy-side").into_bytes(); - let store = open(&dir).await; - assert!(store.legacy_only_keys().contains(&key)); - - // Intact: the offer is correctly refused. - assert!(store.holds_verified(&key, &content).await); - - // Wreck the legacy copy underneath, leaving the key in the union view. - let legacy = store.legacy().expect("legacy"); - legacy.lmdb.delete(&key).await.expect("delete"); - assert!( - store.holds_verified(&key, &content).await, - "with no readable legacy copy the offered bytes must be taken, not refused" - ); - assert_eq!( - store.get(&key).await.expect("get").expect("present"), - content - ); - assert!( - !store.legacy_only_keys().contains(&key), - "and the key must leave the legacy-only set now that a file holds it" - ); - } - - /// A chunk this node does not have is not claimed as held. - #[tokio::test] - async fn a_chunk_this_node_does_not_have_is_not_claimed() { - let dir = TempDir::new().expect("temp dir"); - let store = open(&dir).await; - let (addr, content) = addressed("never-stored"); - assert!(!store.holds_verified(&addr, &content).await); - } - - /// An environment is never deleted because it failed to open. - /// - /// Opening is not a corruption test. It queries free space, maps the file, takes a - /// write transaction and scans every key, so a full disk, a permission change or a - /// transient fault all look identical to corruption. The only thing that counts is the - /// directory's own mark. - #[tokio::test] - async fn an_unmarked_environment_is_kept_however_badly_it_reads() { - let dir = TempDir::new().expect("temp dir"); - seed_legacy(&dir, &["unmarked"]).await; - let env = dir.path().join(LEGACY_ENV_DIR); - assert_eq!(retirement_mark(&env), RetirementMark::Absent); + write_temp(temp_path, payload)?; + // Test-only: the one moment a complete chunk exists on disk under a name nothing + // looks for. A crash test needs to die at a named point rather than wherever a + // sleep in another process happened to land. + #[cfg(any(test, feature = "test-utils"))] + halt_here_if_asked(HALT_BEFORE_PUBLISH, temp_path); + match rename_with_retry(temp_path, final_path) { + Ok(()) => PutOutcome::New, + Err(e) => { + let _ = std::fs::remove_file(temp_path); + // Another writer of the same address won the race, or the destination was + // open. Either way the bytes are already published. + if !final_path.exists() { + return Err(Error::Storage(format!( + "Failed to publish chunk {}: {e}", + final_path.display() + ))); + } + PutOutcome::Duplicate + } + } + }; - finish_interrupted_retirement(dir.path()); - assert!( - env.exists(), - "an environment carrying no retirement mark must never be removed" - ); - } + Ok(outcome) +} - /// A directory carrying its own retirement mark is finished off, whatever it is named. - /// - /// This is the case the mark exists for. Off Unix the rename that moves the - /// environment aside cannot be shown to be durable, so a power loss can bring it back - /// under its old name with its contents already deleted. Without the mark the node - /// would refuse to start on it forever; with it, the directory says what it is. - #[tokio::test] - async fn a_directory_that_says_it_was_retired_is_removed_under_any_name() { - let dir = TempDir::new().expect("temp dir"); - let keys = seed_legacy(&dir, &["reverted"]).await; - { - let store = open(&dir).await; - store - .copy_batch(&keys, 0, 0, &never_cancelled()) - .await - .expect("copy"); - } - // What a reverted rename leaves: the old name, the retirement mark inside it. - let env = dir.path().join(LEGACY_ENV_DIR); - mark_directory_retired(&env).expect("mark"); +/// A publish that failed, and whether it left its bytes on the disk. +/// +/// The second half is the point. A failure before anything was created has spent nothing; +/// one that created the file and then could not remove it again has spent the space, and +/// whoever is accounting for free space has to know which happened. Only the code that did +/// the creating can say. +struct PublishFailed { + error: Error, + left_behind: bool, +} - let store = open(&dir).await; - assert!( - !store.has_legacy(), - "the remains of an interrupted removal must not be adopted" - ); - assert!(!env.exists(), "and the next start must finish the removal"); - // Every chunk is still served, from the file store. - for key in &keys { - assert!(store.get(key).await.expect("get").is_some()); +impl PublishFailed { + /// A failure that created nothing. + fn nothing_written(error: Error) -> Self { + Self { + error, + left_behind: false, } } +} - /// The mark goes inside the directory, so a rename cannot separate them. - /// - /// A mark beside the environment would have to be cancelled when a retirement is - /// abandoned, cancellation can fail or be lost, and a stale one would then authorise - /// deleting an environment that had since taken a chunk. - #[test] - fn the_retirement_mark_travels_with_the_directory() { - let dir = TempDir::new().expect("temp dir"); - let original = dir.path().join("chunks.mdb"); - std::fs::create_dir_all(&original).expect("mkdir"); - mark_directory_retired(&original).expect("mark"); - assert_eq!(retirement_mark(&original), RetirementMark::Present); +/// Make a publication durable by flushing the directory its name lives in. +/// +/// Separate from placing the bytes, because the caller has to tell the two failures apart. +/// A publish that fails before the bytes land has spent nothing; one that fails here has +/// spent the space and must not be reported as stored, so whoever is accounting for free +/// space has to charge it while whoever is accounting for chunks must not count it. +/// +/// NOT best effort. The directory flush is what makes the rename durable, and a copy +/// reported successful is what authorises deleting the only other copy. Swallowing the +/// failure would let a power loss discard the directory entry after the legacy store had +/// already been removed. +/// +/// # Errors +/// +/// Returns [`Error::Storage`] if the directory cannot be flushed. +#[cfg(unix)] +fn flush_publication(final_path: &Path, shard: &Path) -> Result<()> { + fsync_dir(shard).map_err(|e| { + Error::Storage(format!( + "Published {} but could not flush {}: {e}. Not reporting this chunk as stored, \ + because a copy that is not durable must not authorise deleting another.", + final_path.display(), + shard.display() + )) + }) +} - let renamed = dir.path().join("chunks.mdb.retired"); - std::fs::rename(&original, &renamed).expect("rename"); - assert!( - retirement_mark(&renamed) == RetirementMark::Present, - "the mark must survive the rename it exists to outlive" - ); - // And back again, which is what a power loss undoing the rename looks like. - std::fs::rename(&renamed, &original).expect("rename back"); - assert_eq!(retirement_mark(&original), RetirementMark::Present); - } +/// Nothing to do off Unix, where the chunk is created under its final name and flushed +/// with `sync_all`, which is documented to carry its creation metadata with it, and where +/// there is no way to flush a directory at all. +#[cfg(not(unix))] +fn flush_publication(_final_path: &Path, _shard: &Path) -> Result<()> { + Ok(()) +} + +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +mod tests { + use super::*; + use std::collections::HashSet; - /// Losing the handle to an environment that is still there is not completion. + /// A directory flush that fails must say so. /// - /// A rename that failed and then could not be reopened leaves the directory on disk - /// with no way to read it. Treating the missing handle as "nothing left to migrate" - /// would have the driver log the migration finished over a store still holding chunks - /// nothing else can serve. - #[tokio::test] - async fn a_lost_handle_beside_a_live_environment_blocks_retirement() { + /// The quiet version of this function is only used where the answer does not change + /// what happens next. On the publish path it does. + #[cfg(unix)] + #[test] + fn flushing_a_directory_that_is_not_there_reports_the_failure() { let dir = TempDir::new().expect("temp dir"); - seed_legacy(&dir, &["orphaned"]).await; - let store = open(&dir).await; - assert!(store.has_legacy()); - - // Stand in for a failed rename followed by a failed reopen. - *store.legacy.write() = None; - assert!(!store.has_legacy()); - assert!(dir.path().join(LEGACY_ENV_DIR).exists()); - - let blocker = store - .retirement_blocker(|_| false) - .expect("a live environment with no handle must block"); - assert!( - blocker.contains("no handle"), - "the reason must name the actual problem, got: {blocker}" - ); + assert!(fsync_dir(dir.path()).is_ok()); + assert!(fsync_dir(&dir.path().join("no-such-shard")).is_err()); } - /// A node that still holds its store open is gated too. + /// A chunk whose directory entry was never flushed is not reported as stored. /// - /// The classification used to be asked only when there was no handle, which meant the - /// ordinary path never asked it: a node holding its environment open went through every - /// gate, renamed the directory aside and deleted it, whatever the mark said or failed to - /// say. Retirement deletes the last other copy of these chunks, so it is not a question - /// to skip because a different question already had an answer. + /// This is the whole safety argument for retirement: the legacy store is deleted + /// because every chunk was copied durably. A published file whose directory flush + /// failed can vanish on power loss, so counting it as copied would lose data. The + /// file staying on disk afterwards is fine, the next pass republishes it. #[cfg(unix)] - #[tokio::test] - async fn an_environment_that_cannot_be_classified_blocks_retirement_even_with_a_handle() { + #[test] + fn a_publish_whose_directory_flush_fails_is_not_reported_as_stored() { let dir = TempDir::new().expect("temp dir"); - seed_legacy(&dir, &["still-open"]).await; - let store = open(&dir).await; - assert!( - store.has_legacy(), - "this one keeps its handle, deliberately" - ); - // Whatever else is in the way at this point, it is not this. Compared rather than - // required to be nothing, because the other gates have their own tests and their - // own reasons to be unmet here. - let before = store.retirement_blocker(|_| false).unwrap_or_default(); + let temp_path = dir.path().join("chunk.tmp"); + let final_path = dir.path().join("chunk"); + let unflushable = dir.path().join("shard-that-does-not-exist"); + + // Asserted in two steps, not chained. Chaining them means a regression in placing + // the bytes also produces an error, and the test passes without the flush ever + // being reached: it would be checking that something went wrong rather than that + // this went wrong. + let placed = publish_via_rename(&temp_path, &final_path, b"payload", &unflushable); assert!( - !before.contains("already retired"), - "a readable environment must not be blocked for being unclassifiable: {before}" + placed.is_ok(), + "the bytes must be placed before this can be about the flush: {:?}", + placed.err() ); + let outcome = flush_publication(&final_path, &unflushable); - make_the_mark_unreadable(&dir.path().join(LEGACY_ENV_DIR)); - - let blocker = store - .retirement_blocker(|_| false) - .expect("an environment that cannot be classified must block retirement"); assert!( - blocker.contains("already retired"), - "the reason must name the actual problem, got: {blocker}" + outcome.is_err(), + "an unflushed publication must not be reported as stored" ); assert!( - store.legacy_cannot_be_classified(), - "and the driver must see it as work only a person can finish, so that it \ - gives the shared volume back" + !temp_path.exists(), + "the temp file must not be left behind either way" ); } - /// A directory under the retired name with no mark inside it is an intact store. - /// - /// It got that name from a rename, and the rename happens after every gate; the mark - /// is written straight afterwards. A crash in between leaves a whole environment - /// wearing a name that says otherwise, and deleting it because of what it is called - /// would destroy every chunk in it. It is put back instead. - #[tokio::test] - async fn an_unmarked_retired_directory_is_restored_rather_than_deleted() { - let dir = TempDir::new().expect("temp dir"); - let keys = seed_legacy(&dir, &["not-really-retired"]).await; - let env = dir.path().join(LEGACY_ENV_DIR); - let tombstone = dir.path().join(format!("{LEGACY_ENV_DIR}{RETIRED_SUFFIX}")); - std::fs::rename(&env, &tombstone).expect("rename"); - assert_eq!(retirement_mark(&tombstone), RetirementMark::Absent); - - let store = open(&dir).await; - assert!( - store.has_legacy(), - "an unmarked environment must be restored and served, not deleted" - ); - assert!(env.exists(), "it must be back under its own name"); - for key in &keys { - assert!(store.get(key).await.expect("get").is_some()); - } - } + use tempfile::TempDir; - /// A mark already at that name is not a mark until it can be read. - /// - /// The one place in this file that was taking the name as the evidence, which is the - /// thing every other part of it refuses to do. Retirement writes the mark with - /// `create_new`, and a failure saying something is already there was accepted as "the - /// mark is present" and the environment deleted on the strength of it. What is at that - /// name might be anything. - /// - /// It matters most for a node that already has its store open. That path never - /// consulted the mark at all until this round, so an unreadable one would have gone - /// through every gate and been deleted. - #[cfg(unix)] - #[test] - fn a_mark_already_at_that_name_is_not_accepted_until_it_can_be_read() { + /// Open a store on a fresh temp directory with the disk reserve disabled. + async fn test_store() -> (ChunkStore, TempDir) { let dir = TempDir::new().expect("temp dir"); - let env = dir.path().join(LEGACY_ENV_DIR); - std::fs::create_dir_all(&env).expect("mkdir"); - make_the_mark_unreadable(&env); - - let refused = mark_directory_retired(&env).expect_err("an unreadable mark is not a mark"); - assert!( - !refused.mark_definitely_gone, - "something is at that name, so the caller must not treat it as absent and put \ - the directory back under a name that will be opened" - ); - assert_eq!(retirement_mark(&env), RetirementMark::Unknown); + let store = ChunkStore::new(ChunkStoreConfig { + root_dir: dir.path().to_path_buf(), + verify_on_read: true, + disk_reserve: 0, + }) + .await + .expect("open store"); + (store, dir) + } - // And a real one is still accepted, so the refusal above is about being unable to - // read it rather than about there being something there at all. - std::fs::remove_file(env.join(RETIRED_MARKER)).expect("clear the link"); - mark_directory_retired(&env).expect("a first mark"); - mark_directory_retired(&env).expect("and the same mark again, which is readable"); - assert_eq!(retirement_mark(&env), RetirementMark::Present); + /// Open a store on an existing directory, as a restart would. + async fn reopen(dir: &TempDir) -> ChunkStore { + ChunkStore::new(ChunkStoreConfig { + root_dir: dir.path().to_path_buf(), + verify_on_read: true, + disk_reserve: 0, + }) + .await + .expect("reopen store") } - /// A directory nothing can classify is neither restored nor deleted. - /// - /// The half of the three-state answer that a first attempt at this got wrong. Asking - /// only "may it be removed" and letting everything else fall through puts "cannot tell" - /// straight back on the restoring path, which is the resurrection this exists to - /// prevent: the mark check can fail for a moment and succeed the next, and the restore - /// in between brings back a store that really had been retired. + /// An ordinary read settles whether the node answers for a chunk. /// - /// Unix only: the state is staged with a symbolic link, which Windows does not offer - /// on the same terms. + /// Not only the reads that were checking something. A read that failed means the + /// chunk cannot be served, whoever asked; a read that worked means it can be. Deciding + /// this anywhere else leaves a key stuck unadvertised after the fault has cleared, or + /// advertised after it has not. #[cfg(unix)] #[tokio::test] - async fn a_tombstone_that_cannot_be_classified_is_left_where_it_is() { - let dir = TempDir::new().expect("temp dir"); - seed_legacy(&dir, &["unclassifiable"]).await; - let env = dir.path().join(LEGACY_ENV_DIR); - let tombstone = dir.path().join(format!("{LEGACY_ENV_DIR}{RETIRED_SUFFIX}")); - std::fs::rename(&env, &tombstone).expect("rename"); - make_the_mark_unreadable(&tombstone); - assert_eq!(retirement_mark(&tombstone), RetirementMark::Unknown); + async fn an_ordinary_read_decides_whether_the_node_answers_for_a_chunk() { + use std::os::unix::fs::PermissionsExt; - sweep_retired_legacy(dir.path()); - let still_there = tombstone.exists(); - let restored = env.exists(); + let (store, dir) = test_store().await; + let (addr, content) = addressed("read-decides"); + store.put(&addr, &content).await.expect("put"); + let path = store.chunk_path(&addr); - assert!( - still_there, - "a directory that cannot be classified must not be deleted: it may be intact" - ); - assert!( - !restored, - "a directory that cannot be classified must not be put back under the live \ - name: it may be half deleted" - ); - } + let mut perms = std::fs::metadata(&path).expect("meta").permissions(); + perms.set_mode(0o000); + std::fs::set_permissions(&path, perms).expect("chmod"); - /// The same directory under the live name is not opened either. - /// - /// Opening it would put keys back into a commitment they may already have left, and - /// this node cannot tell whether they have. Serving from files alone is the answer that - /// is right either way. - #[cfg(unix)] - #[test] - fn a_live_environment_that_cannot_be_classified_is_not_opened() { - let dir = TempDir::new().expect("temp dir"); - let env = dir.path().join(LEGACY_ENV_DIR); - std::fs::create_dir_all(&env).expect("mkdir"); - std::fs::write(env.join("data.mdb"), b"not really an environment").expect("seed"); - assert_eq!( - finish_interrupted_retirement(dir.path()), - LiveEnvironment::WhateverIsOnDisk, - "a readable directory with no mark is ordinary and may be opened" + assert!(store.get(&addr).await.is_err(), "the read must fail"); + assert!( + !store.exists(&addr).expect("exists"), + "and a plain read that failed must stop the node answering for it" ); - make_the_mark_unreadable(&env); - assert_eq!(retirement_mark(&env), RetirementMark::Unknown); - - let verdict = finish_interrupted_retirement(dir.path()); - let still_there = env.exists(); + let mut perms = std::fs::metadata(&path).expect("meta").permissions(); + perms.set_mode(0o600); + std::fs::set_permissions(&path, perms).expect("chmod back"); assert_eq!( - verdict, - LiveEnvironment::None, - "a directory that cannot be classified must not be opened" + store.get(&addr).await.expect("get").expect("present"), + content ); assert!( - still_there, - "and it must not be deleted either: it may be a live environment" + store.exists(&addr).expect("exists"), + "and a plain read that worked must start it answering again" ); + drop(dir); } - /// Make the retirement mark in `dir` unreadable without touching the directory itself. + /// Two writes for one key: waiting means waiting for both. /// - /// A symbolic link pointing at itself. Looking for the mark follows it, gets - /// `FilesystemLoop` back, and the answer is neither "there" nor "not there", which is - /// the state under test. - /// - /// Taking the directory's permissions away instead was the first attempt and staged too - /// much: at mode 000 the operating system refuses the rename as well, so the code being - /// tested was never reached and the test passed with its own protection removed. Root - /// can also read a mode-000 directory, which would have made it fail on any CI that - /// runs as root. A link loop is neither: everything else about the directory keeps - /// working, for every user. - #[cfg(unix)] - fn make_the_mark_unreadable(dir: &Path) { - let link = dir.join(RETIRED_MARKER); - let _ = std::fs::remove_file(&link); - std::os::unix::fs::symlink(&link, &link).expect("a link to itself"); - } - - /// Both names taken is not a decision this code makes. + /// Cancellation releases the caller's lane while the blocking half survives, so a + /// second write for the same key can start behind the first. If the registry only + /// recorded that *something* was writing, whichever finished first would clear it and + /// a delete would be told the key was free while the other was still queued, then be + /// undone by it. #[tokio::test] - async fn an_unmarked_retired_directory_beside_a_live_one_is_left_alone() { - let dir = TempDir::new().expect("temp dir"); - seed_legacy(&dir, &["live"]).await; - let tombstone = dir.path().join(format!("{LEGACY_ENV_DIR}{RETIRED_SUFFIX}")); - std::fs::create_dir_all(&tombstone).expect("mkdir"); - std::fs::write(tombstone.join("data.mdb"), b"something").expect("write"); - - let store = open(&dir).await; - assert!(store.has_legacy()); - assert!( - tombstone.exists(), - "an unmarked directory must never be deleted, even beside a live one" - ); - assert!(dir.path().join(LEGACY_ENV_DIR).exists()); - } - - /// The mark is the last thing a deletion takes away. - /// - /// A recursive delete walks in whatever order the filesystem gives, so it can unlink - /// the mark and then fail on the next entry, which is what a sharing violation on the - /// data file looks like. That leaves a genuinely retired, partly deleted directory - /// carrying no evidence of it, and the next start would read that as intact and - /// restore it. - /// Unix only: the failure is provoked with directory permissions, which is not how - /// the same thing happens on Windows. The behaviour under test is platform-neutral. - #[cfg(unix)] - #[test] - fn a_failed_deletion_leaves_the_mark_in_place() { - use std::os::unix::fs::PermissionsExt; + async fn waiting_for_a_key_waits_for_every_write_of_it() { + let (store, dir) = test_store().await; + let store = Arc::new(store); + let (addr, content) = addressed("two-writers"); - let dir = TempDir::new().expect("temp dir"); - let retired = dir.path().join("chunks.mdb.retired"); - std::fs::create_dir_all(&retired).expect("mkdir"); - std::fs::write(retired.join(LEGACY_DATA_FILE), b"payload").expect("write"); - mark_directory_retired(&retired).expect("mark"); - - // An entry that cannot be removed, standing in for whatever the filesystem - // refuses on the day. - std::fs::create_dir_all(retired.join("stuck")).expect("mkdir"); - let mut perms = std::fs::metadata(&retired).expect("meta").permissions(); - perms.set_mode(0o500); - std::fs::set_permissions(&retired, perms.clone()).expect("chmod"); + // Two registrations, as two overlapping writes would make. + let first = store.begin_write(&addr); + let second = store.begin_write(&addr); - let failed = remove_marked_directory(&retired).is_err(); - - perms.set_mode(0o700); - std::fs::set_permissions(&retired, perms).expect("chmod back"); + let waiting = { + let store = Arc::clone(&store); + tokio::spawn(async move { store.wait_for_write(&addr).await }) + }; + tokio::time::sleep(Duration::from_millis(50)).await; + assert!(!waiting.is_finished()); - assert!(failed, "the deletion was supposed to fail"); + // One finishes. The other has not, so the wait must continue. + drop(first); + tokio::time::sleep(Duration::from_millis(50)).await; assert!( - retirement_mark(&retired) == RetirementMark::Present, - "a deletion that failed must leave the mark, or the directory stops saying \ - what it is" + !waiting.is_finished(), + "one write finishing does not mean the key is free" ); + + drop(second); + waiting + .await + .expect("the wait ends once both have finished"); + + // And the store is still usable afterwards. + store.put(&addr, &content).await.expect("put"); + assert!(store.exists(&addr).expect("exists")); + drop(dir); } - /// A mark that cannot be read is not permission to do anything. + /// A chunk this store cannot read is kept but not claimed. /// - /// The reason this is three states and not two. Reading the mark can fail for reasons - /// that are neither yes nor no, and the old answer for those was "no mark", which is - /// the worst of the three: a retired environment reads as live, goes back under its own - /// name, and its keys re-enter a commitment they have already left. Deleting on an - /// unreadable answer would be just as wrong in the other direction. - /// - /// Unix only: the state is staged with a symbolic link, which Windows does not offer - /// on the same terms. + /// Both halves matter. Deleting it, or dropping it from the index, is how a chunk ends + /// up in neither this store's view nor the legacy one, which is what retirement + /// destroys. Claiming it anyway puts the key in signed commitments and answers + /// presence probes with a yes for a chunk the node cannot serve, and the audit that + /// catches that still penalises. #[cfg(unix)] - #[test] - fn a_mark_that_cannot_be_read_permits_nothing() { - let dir = TempDir::new().expect("temp dir"); - let env = dir.path().join(LEGACY_ENV_DIR); - - // Nothing there is not the same as cannot tell, and it is the ordinary case: every - // node that never had a legacy store, and every node that has finished with one, - // asks this question on every tick. Answering "cannot tell" for those would have a - // fresh node run a migration driver forever over a store it does not have. - assert_eq!( - retirement_mark(&env), - RetirementMark::Absent, - "a directory that is not there carries no mark, and that is known" - ); - assert!(retirement_mark(&env).permits_opening()); + #[tokio::test] + async fn a_chunk_that_cannot_be_read_is_kept_but_not_claimed() { + use std::os::unix::fs::PermissionsExt; - std::fs::create_dir_all(&env).expect("mkdir"); - std::fs::write(env.join(RETIRED_MARKER), b"retired").expect("mark"); - assert_eq!(retirement_mark(&env), RetirementMark::Present); + let (store, dir) = test_store().await; + let (addr, content) = addressed("unreadable-for-now"); + store.put(&addr, &content).await.expect("put"); + assert!(store.exists(&addr).expect("exists")); - // Looking for the mark now goes round in a circle, so the answer is neither there - // nor not there. - make_the_mark_unreadable(&env); - let unreadable = retirement_mark(&env); + let path = store.chunk_path(&addr); + let mut perms = std::fs::metadata(&path).expect("meta").permissions(); + perms.set_mode(0o000); + std::fs::set_permissions(&path, perms).expect("chmod"); - assert_eq!( - unreadable, - RetirementMark::Unknown, - "a mark that cannot be read must not report as absent" - ); + // Offering the same bytes again must not be acknowledged, and must not replace + // what is there on the strength of a read that did not happen. assert!( - !unreadable.permits_removal(), - "an unreadable mark must not authorise deleting the environment" + store.put(&addr, &content).await.is_err(), + "an unreadable chunk must not be reported as stored" ); + assert!(path.exists(), "and the file must be left alone"); assert!( - !unreadable.permits_opening(), - "an unreadable mark must not authorise reopening the environment" + !store.exists(&addr).expect("exists"), + "but the node must stop claiming it" ); + assert!(!store.all_keys().await.expect("keys").contains(&addr)); + + // Readable again: the node answers for it once more. + let mut perms = std::fs::metadata(&path).expect("meta").permissions(); + perms.set_mode(0o600); + std::fs::set_permissions(&path, perms).expect("chmod back"); + assert!(!store.put(&addr, &content).await.expect("put again")); + assert!(store.exists(&addr).expect("exists")); + assert!(store.all_keys().await.expect("keys").contains(&addr)); + drop(dir); } - /// A directory under the live name that says it was retired is never opened. - /// - /// It may be partly deleted, and opening it would put keys back into a commitment - /// they have already left. Its chunks are in the file store, which is what the mark - /// records, so serving from files alone is correct. - #[tokio::test] - async fn a_marked_directory_under_the_live_name_is_not_served_from() { - let dir = TempDir::new().expect("temp dir"); - let keys = seed_legacy(&dir, &["marked-live"]).await; - { - let store = open(&dir).await; - store - .copy_batch(&keys, 0, 0, &never_cancelled()) - .await - .expect("copy"); - } - let env = dir.path().join(LEGACY_ENV_DIR); - mark_directory_retired(&env).expect("mark"); - - // Every name it could be moved to is taken by something that is not empty, so the - // rename fails and the marked directory stays under the live name. That is the - // case this is about: it must be left alone rather than opened. - for n in 0..=MAX_TOMBSTONES { - let taken = if n == 0 { - dir.path().join(format!("{LEGACY_ENV_DIR}{RETIRED_SUFFIX}")) - } else { - dir.path() - .join(format!("{LEGACY_ENV_DIR}{RETIRED_SUFFIX}.{n}")) + /// Content plus the address it hashes to. + /// Every path under `root`, sorted, so a test can say the disk did not change. + fn tree_under(root: &Path) -> Vec { + let mut found = Vec::new(); + let mut stack = vec![root.to_path_buf()]; + while let Some(dir) = stack.pop() { + let Ok(entries) = std::fs::read_dir(&dir) else { + continue; }; - std::fs::create_dir_all(&taken).expect("mkdir"); - std::fs::write(taken.join("occupied"), b"x").expect("write"); + for entry in entries.flatten() { + let path = entry.path(); + if entry.file_type().is_ok_and(|t| t.is_dir()) { + stack.push(path.clone()); + } + found.push(path); + } } + found.sort(); + found + } - let store = open(&dir).await; - assert!( - env.exists(), - "the rename was supposed to fail, leaving the marked directory in place" - ); - assert!( - !store.has_legacy(), - "a directory that says it was retired must never be opened as live" - ); - for key in &keys { - assert!(store.get(key).await.expect("get").is_some()); - } + fn addressed(seed: &str) -> (XorName, Vec) { + let content = format!("chunk-content-{seed}").into_bytes(); + (crate::client::compute_address(&content), content) } - /// A linked environment is copied out of but never deleted. + /// A value over the ceiling is not a chunk, and this build will not hold one anywhere. /// - /// An operator who points the chunk store at another volume leaves a link here. - /// Retirement renames the path and then deletes what is behind it, and behind a link - /// is a directory somewhere else that this node does not own. - #[cfg(unix)] + /// The 4 MB ceiling is what makes a chunk a chunk, and every released ingress enforces it + /// before anything is stored: the protocol handler on a paid store, and replication on + /// both the receive and the fetch path. So no over-ceiling value ever entered this network + /// as a chunk, and one found on a disk is not data to be preserved — it is not a chunk at + /// all. + /// + /// There was exactly one way to get one onto a disk, and it was ours, not the network's. + /// The bridge's `put` wrote to the legacy environment FIRST — LMDB has no size ceiling — + /// and only then offered the same bytes to the file store, which refused them. It then + /// recorded the key as legacy-only so the copier would retry it, and the copier's own + /// size arm deleted it. A local caller of the public `ChunkStore::put` was the whole of + /// the exposure. + /// + /// That path is gone with the bridge. There is one store, its `put` refuses over the + /// ceiling, and it refuses BEFORE it writes anything, so there is no half-written state + /// to reason about and nothing for a later pass to find and have to decide about. This is + /// why the release preserves no such value and grows no sidecar to put one in: there is + /// no valid chunk to lose, and building a place to keep invalid ones would be building + /// for a case this release makes unreachable. #[tokio::test] - async fn a_linked_environment_is_never_retired() { - let outside = TempDir::new().expect("temp dir"); - let dir = TempDir::new().expect("temp dir"); - // A real environment that lives in `outside`; the node root only links to it. - seed_legacy(&outside, &["someone-elses"]).await; - let real = outside.path().join(LEGACY_ENV_DIR); - let bystander = outside.path().join("unrelated"); - std::fs::create_dir_all(&bystander).expect("mkdir"); - std::os::unix::fs::symlink(&real, dir.path().join(LEGACY_ENV_DIR)).expect("symlink"); - - let store = open(&dir).await; - let blocker = store - .retirement_blocker(|_| false) - .expect("a linked environment must block retirement"); + async fn a_value_over_the_ceiling_is_refused_before_anything_is_written() { + let (store, dir) = test_store().await; + + // Addressed to its own bytes, so the content-address arm passes and the refusal is + // the size one. Checking the wrong arm would pass this test while the ceiling was + // gone. + let content = vec![0xAB; MAX_CHUNK_SIZE + 1]; + let address = crate::client::compute_address(&content); + + // Every path under the store before the call, so "nothing was written" is asserted + // about the whole tree and not only about the name this chunk would have had. A + // regression that wrote a temporary somewhere else and tidied it up again would pass + // the narrower check. + let before = tree_under(dir.path()); + + let err = store + .put(&address, &content) + .await + .expect_err("a value over the ceiling is not a chunk and must be refused"); + let message = format!("{err}"); assert!( - blocker.contains("link"), - "the reason must name the actual problem, got: {blocker}" + message.contains("byte maximum"), + "refused for the wrong reason, so this proves nothing about the ceiling: {message}" ); - // And nothing walks through it, whatever it is marked with. - std::fs::write(real.join(RETIRED_MARKER), b"x").expect("mark through the link"); assert!( - retirement_mark(&dir.path().join(LEGACY_ENV_DIR)) != RetirementMark::Present, - "a link must never be treated as a retired directory" + !store.exists(&address).expect("exists"), + "the node must not claim a value it refused" ); + assert!(!store.all_keys().await.expect("keys").contains(&address)); assert!( - remove_marked_directory(&dir.path().join(LEGACY_ENV_DIR)).is_err(), - "deleting through a link must be refused" + !store.chunk_path(&address).exists(), + "nothing may be left on disk: the refusal comes before the write" ); - assert!( - real.join(LEGACY_DATA_FILE).exists(), - "and must delete nothing" + assert_eq!( + tree_under(dir.path()), + before, + "the refusal touched the disk; it is supposed to come before anything is written" ); - assert!(bystander.exists()); - } - - /// A deletion that cannot remove the directory itself puts the mark back. - /// - /// An unmarked directory that still exists is the one state the scheme says cannot - /// happen: the next start would read it as an intact environment. - /// - /// Unix only: the failure is provoked with directory permissions, which is not how - /// the same thing happens on Windows. The behaviour under test is platform-neutral. - #[cfg(unix)] - #[test] - fn a_directory_that_cannot_be_removed_keeps_saying_it_was_retired() { - use std::os::unix::fs::PermissionsExt; - let dir = TempDir::new().expect("temp dir"); - let retired = dir.path().join("chunks.mdb.retired"); - std::fs::create_dir_all(&retired).expect("mkdir"); - std::fs::write(retired.join(LEGACY_DATA_FILE), b"payload").expect("write"); - mark_directory_retired(&retired).expect("mark"); - - // The parent read-only, so the directory cannot be unlinked from it while its own - // contents still can be. That is the shape that leaves an emptied, unmarked - // directory behind. - let mut perms = std::fs::metadata(dir.path()).expect("meta").permissions(); - perms.set_mode(0o500); - std::fs::set_permissions(dir.path(), perms).expect("chmod"); - - let failed = remove_marked_directory(&retired).is_err(); - - let mut perms = std::fs::metadata(dir.path()).expect("meta").permissions(); - perms.set_mode(0o700); - std::fs::set_permissions(dir.path(), perms).expect("chmod back"); - - assert!(failed, "the removal was supposed to fail"); - assert!(retired.exists(), "and to leave the directory behind"); - assert!( - retirement_mark(&retired) == RetirementMark::Present, - "a directory that outlived its deletion must still say what it is" + // And a restart cannot find one either, which is what says the refusal left no + // partial file for a startup scan to index by name. The store holds its directory + // lock for as long as it lives, so it goes first: two of them on one root is refused, + // which is a different failure and would prove nothing about the ceiling. + drop(store); + let reopened = reopen(&dir).await; + assert!(!reopened.exists(&address).expect("exists")); + assert_eq!( + reopened.current_chunks().expect("count"), + 0, + "a refused value must not survive as a name the scan believes" ); } - /// A key the environment holds that is in neither view stops retirement. - /// - /// The gates only ever see the legacy-only set, so a key that has fallen out of both - /// the file index and that set has been through nothing and is protected by nothing. - /// It is the environment's only copy, and retirement would take it. #[tokio::test] - async fn a_legacy_key_in_neither_view_refuses_the_proof_and_is_re_queued() { - let dir = TempDir::new().expect("temp dir"); - let keys = seed_legacy(&dir, &["orphan"]).await; - let key = *keys.first().expect("one key"); - let store = open(&dir).await; - store - .copy_batch(&keys, 0, 0, &never_cancelled()) - .await - .expect("copy"); - assert!(!store.legacy_only_keys().contains(&key)); + async fn put_then_get_returns_the_same_bytes() { + let (store, _dir) = test_store().await; + let (addr, content) = addressed("a"); - // Stand in for whatever takes the file out from under the index: a quarantine, a - // publish that failed, an operator. The key is now in neither view. - let path = dir - .path() - .join("chunks") - .join(format!("{:02x}", key.last().copied().unwrap_or(0))) - .join(hex::encode(key)); - std::fs::remove_file(&path).expect("remove the file"); - store.files.forget_for_test(&key); - assert!(!store.files.exists(&key).unwrap_or(false)); - assert!(!store.legacy_only_keys().contains(&key)); - - let proof = store - .verify_before_retire(0, &never_cancelled()) - .await - .expect("verify"); - assert!( - !proof.is_clean(), - "a key protected by neither view must refuse the proof" - ); - assert!( - store.legacy_only_keys().contains(&key), - "and must be put back where the gates can see it" - ); + assert!(store.put(&addr, &content).await.expect("put")); + let got = store.get(&addr).await.expect("get").expect("present"); + assert_eq!(got, content); } - /// A verification that no longer describes the store does not authorise a deletion. - /// - /// The pass reads every chunk and its result is reused for a while rather than re-read - /// on every tick. Retirement is often deferred in that window by a gate that has - /// nothing to do with the files. If a kept chunk stops being readable meanwhile, - /// ordinary requests are still served from the legacy copy, and deleting that copy on - /// the strength of the older pass leaves the node holding only the unreadable one. - #[cfg(unix)] #[tokio::test] - async fn a_verification_overtaken_by_a_failing_file_does_not_authorise_retirement() { - use std::os::unix::fs::PermissionsExt; - - let dir = TempDir::new().expect("temp dir"); - let keys = seed_legacy(&dir, &["kept-then-unreadable"]).await; - let key = *keys.first().expect("one key"); - let mut config = test_config(&dir); - config.migration.retire_legacy = true; - let store = ChunkStore::new(config).await.expect("open"); - store - .copy_batch(&keys, 0, 0, &never_cancelled()) - .await - .expect("copy"); - open_the_retirement_gate(&store); + async fn a_second_put_of_the_same_chunk_reports_not_new() { + let (store, _dir) = test_store().await; + let (addr, content) = addressed("b"); - let proof = store - .verify_before_retire(0, &never_cancelled()) - .await - .expect("verify"); - assert!(proof.is_clean()); + assert!(store.put(&addr, &content).await.expect("first put")); + assert!(!store.put(&addr, &content).await.expect("second put")); + assert_eq!(store.current_chunks().expect("count"), 1); + assert_eq!(store.stats().duplicates, 1); + } - // The window: the file stops being readable after the pass and before the - // deletion it authorised. - let path = dir - .path() - .join("chunks") - .join(format!("{:02x}", key.last().copied().unwrap_or(0))) - .join(hex::encode(key)); - let mut perms = std::fs::metadata(&path).expect("meta").permissions(); - perms.set_mode(0o000); - std::fs::set_permissions(&path, perms).expect("chmod"); - assert!( - store.get(&key).await.is_ok(), - "the legacy copy still serves it, which is what hides the problem" - ); + #[tokio::test] + async fn get_of_an_unknown_address_is_none() { + let (store, _dir) = test_store().await; + let (addr, _) = addressed("missing"); + assert!(store.get(&addr).await.expect("get").is_none()); + } - let err = store - .retire_legacy(&proof, &|_: &XorName| false, &approved_shed(&store)) - .await - .expect_err("a verification the store has outrun must not authorise a delete"); - assert!(format!("{err}").contains("no longer describes"), "{err}"); - assert!( - store.has_legacy(), - "and the legacy environment must survive" - ); + #[tokio::test] + async fn exists_tracks_the_store() { + let (store, _dir) = test_store().await; + let (addr, content) = addressed("c"); - let mut perms = std::fs::metadata(&path).expect("meta").permissions(); - perms.set_mode(0o600); - std::fs::set_permissions(&path, perms).expect("chmod back"); + assert!(!store.exists(&addr).expect("exists")); + store.put(&addr, &content).await.expect("put"); + assert!(store.exists(&addr).expect("exists")); + store.delete(&addr).await.expect("delete"); + assert!(!store.exists(&addr).expect("exists")); } - /// A write with no rollback copy is counted, on the path that actually skips. - /// - /// The environment is pinned to its current size for the whole bridge, so one with no - /// reusable page answers `Full` to every write and the node stores in files alone. That - /// is accepted, and the ADR says so. What was missing was any way to ask how often it - /// happens: the second release turns on knowing how many nodes are really keeping a - /// rollback copy, and a log line per chunk does not answer it. - /// - /// The first version of this counter missed exactly this path and counted only the - /// other one, the write that is attempted and refused. On the node most affected the - /// other path never runs, so the counter stayed at zero on precisely the nodes it was - /// added for. #[tokio::test] - async fn a_write_with_no_rollback_copy_is_counted() { - let dir = TempDir::new().expect("temp dir"); - seed_legacy(&dir, &["settled"]).await; - let store = open(&dir).await; - assert_eq!( - store.writes_without_a_rollback_copy(), - 0, - "nothing has been skipped yet" - ); + async fn delete_unlinks_the_file_and_returns_the_space() { + let (store, _dir) = test_store().await; + let (addr, content) = addressed("d"); + store.put(&addr, &content).await.expect("put"); - let legacy = store.legacy().expect("legacy"); - let before = legacy - .skipped_rollback_copies - .load(std::sync::atomic::Ordering::Relaxed); - - // Whichever way the environment refuses, the count moves. Driven through the - // counter itself rather than by filling a real environment, because what is under - // test is that the skip is recorded, and staging a genuinely unwritable LMDB from - // here would be testing LMDB. - for _ in 0..12 { - let _ = legacy.note_skipped_rollback_copy(); - } - assert_eq!( - store.writes_without_a_rollback_copy(), - before + 12, - "skipped writes must be visible to whoever asks the node" - ); + let path = store.chunk_path(&addr); + assert!(path.exists(), "the chunk file should be on disk"); - // And the log is throttled, or a node with no free pages writes one line per chunk - // for the rest of its life. - let said: Vec = (0..1_000) - .filter_map(|_| legacy.note_skipped_rollback_copy()) - .collect(); + assert!(store.delete(&addr).await.expect("delete")); + assert!(!path.exists(), "delete must actually unlink the file"); + assert_eq!(store.current_chunks().expect("count"), 0); + + // Deleting again is a no-op that reports nothing was there. + assert!(!store.delete(&addr).await.expect("second delete")); + } + + #[tokio::test] + async fn content_that_does_not_hash_to_its_address_is_rejected() { + let (store, _dir) = test_store().await; + let (addr, _) = addressed("e"); + let err = store + .put(&addr, b"different content") + .await + .expect_err("must reject"); assert!( - said.len() < 10, - "the warning fired {} times in a thousand writes", - said.len() + format!("{err}").contains("Content address mismatch"), + "unexpected error: {err}" ); + assert_eq!(store.current_chunks().expect("count"), 0); } - /// Two writes for one key need two notes, not one shared between them. - /// - /// The journal used to be a set, so a second write for the same key announced nothing - /// and the first to return cleared the entry for both. A delete arriving in that window - /// sees no announcement, skips draining the environment, and the surviving write lands - /// afterwards and puts the key back, undoing a prune the node had decided on. The key - /// then sits in the environment and in neither view, which is the state retirement is - /// built to refuse: no data is lost, but a prune and a retirement cycle are. - /// - /// The file store's own in-flight map is counted for exactly this reason. This is the - /// same reasoning applied to the half that did not have it. #[tokio::test] - async fn two_writes_for_one_key_are_two_notes_and_the_first_to_return_clears_neither() { - let dir = TempDir::new().expect("temp dir"); - seed_legacy(&dir, &["settled"]).await; - let store = open(&dir).await; - let legacy = store.legacy().expect("legacy"); - let (addr, _) = addressed("two-writes"); - - legacy.announce(&addr); - legacy.announce(&addr); - assert!(store.has_pending_writes()); + async fn a_chunk_is_filed_under_the_last_two_hex_characters_of_its_address() { + let (store, dir) = test_store().await; + let (addr, content) = addressed("f"); + store.put(&addr, &content).await.expect("put"); - // The first write returns. The second is still out there, so the note has to stand. - legacy.announced_write_finished(&addr); - assert!( - store.has_pending_writes(), - "the first write to return cleared a note the second one was still relying on" - ); + let name = hex::encode(addr); + let expected_shard = name + .get(name.len() - 2..) + .expect("64-character name") + .to_string(); + let path = dir + .path() + .join(CHUNKS_DIR_NAME) + .join(&expected_shard) + .join(&name); + assert!(path.exists(), "expected the chunk at {}", path.display()); + } - // And the second clears it. - legacy.announced_write_finished(&addr); - assert!(!store.has_pending_writes()); + #[tokio::test] + async fn the_index_is_rebuilt_from_the_filesystem_on_restart() { + let (store, dir) = test_store().await; + let mut written = Vec::new(); + for i in 0..64 { + let (addr, content) = addressed(&format!("restart-{i}")); + store.put(&addr, &content).await.expect("put"); + written.push(addr); + } + drop(store); - // Retiring one that was never announced changes nothing, which is what makes the - // delete path's unconditional clear safe. - legacy.announced_write_finished(&addr); - assert!(!store.has_pending_writes()); + let reopened = reopen(&dir).await; + assert_eq!(reopened.current_chunks().expect("count"), 64); + for addr in &written { + assert!(reopened.exists(addr).expect("exists"), "lost a key"); + } } - /// A write in flight is a note to self, not a claim to hold the chunk. - /// - /// The note exists because a write into the environment outlives the future waiting - /// for it, so a cancelled one could leave a chunk nothing had recorded. But until both - /// halves have returned the node does not hold it, and saying it does puts the key in - /// signed commitments and in the count a quote is priced from. #[tokio::test] - async fn a_write_in_flight_is_not_claimed_but_does_stop_retirement() { - let dir = TempDir::new().expect("temp dir"); - seed_legacy(&dir, &["settled"]).await; - let store = open(&dir).await; - let legacy = store.legacy().expect("legacy"); - let (addr, _) = addressed("in-flight"); + async fn all_keys_is_sorted_ascending() { + let (store, dir) = test_store().await; + for i in 0..128 { + let (addr, content) = addressed(&format!("sorted-{i}")); + store.put(&addr, &content).await.expect("put"); + } - // Stand in for a write that announced itself and never came back. - legacy.announce(&addr); + let keys = store.all_keys().await.expect("all_keys"); + let mut sorted = keys.clone(); + sorted.sort_unstable(); + assert_eq!(keys, sorted, "all_keys() must be ordered"); - assert!( - !store.exists(&addr).expect("exists"), - "a write in flight must not be reported as held" - ); - assert!(!store.all_keys().await.expect("keys").contains(&addr)); - assert!(!store.legacy_only_keys().contains(&addr)); - assert!(store.has_pending_writes()); + // And the order has to survive a restart, because the commitment builder + // truncates the responsible subset before the Merkle tree sorts it. + drop(store); + let reopened = reopen(&dir).await; + assert_eq!(reopened.all_keys().await.expect("all_keys"), keys); + } - // The distinction is the point: the same key in the key set IS claimed. If a - // write announced itself there instead, every one of the assertions above would - // be the opposite for as long as the write took. - legacy.only.write().insert(addr); - assert!(store.exists(&addr).expect("exists")); - assert!(store.all_keys().await.expect("keys").contains(&addr)); - legacy.only.write().remove(&addr); - - // But it does stop the environment going, because what it holds is unsettled. - store.commit_to_files().expect("commit"); - store.note_commitment_rebuilt(); - store.note_commitment_rebuilt(); - store.force_migration_state(|s| { - s.committed_at_unix = - Some(now_unix().saturating_sub(MIN_RETIRE_DELAY_HOURS * 3600 + 60)); - }); - let proof = store - .verify_before_retire(0, &never_cancelled()) - .await - .expect("verify"); - let err = store - .retire_legacy(&proof, &|_: &XorName| false, &approved_shed(&store)) - .await - .expect_err("an unsettled write must stop the removal"); - assert!(format!("{err}").contains("not reported back"), "{err}"); + #[tokio::test] + async fn get_raw_skips_verification() { + let (store, _dir) = test_store().await; + let (addr, content) = addressed("raw"); + store.put(&addr, &content).await.expect("put"); - // And the note is resolved against what is actually there: nothing, so it goes. - store.reconcile_pending_writes().await; - assert!(!store.has_pending_writes()); - assert!(!store.legacy_only_keys().contains(&addr)); + // Corrupt the file behind the store's back. + std::fs::write(store.chunk_path(&addr), b"tampered").expect("tamper"); + + let raw = store.get_raw(&addr).await.expect("get_raw").expect("bytes"); + assert_eq!(raw, b"tampered"); } - /// A delete outlasts a write for the same key that nobody waited for. + /// A delete waits for a put that is already under way for the same key. + /// + /// The narrower ordering, and the one waiting for registered writes does not cover. A + /// put does a lot before it registers itself: it checks the address, reads to see + /// whether the name is taken, and reserves capacity. A delete arriving in that window + /// would see nothing registered, wait for nothing, and go ahead; the put would register + /// and publish afterwards, and the node would keep a chunk it had decided to prune. /// - /// A write has an environment half and a file half, and either can still be running - /// when its caller is dropped: the blocking work is not cancelled with the future. - /// A delete that did not wait for both would be undone by whichever half landed - /// afterwards, putting back a chunk the node had decided to prune. + /// Staged deterministically rather than by racing two tasks. A put parked inside its + /// own closure is holding the key's lane, so the delete must not be able to finish + /// while it is parked. An earlier version of this test started both and accepted either + /// ordering, which the bug also satisfies: it proved nothing. #[tokio::test] - async fn a_delete_outlasts_a_write_nobody_waited_for() { + #[allow(clippy::await_holding_lock)] + async fn a_delete_waits_for_a_put_already_under_way() { let dir = TempDir::new().expect("temp dir"); - seed_legacy(&dir, &["neighbour"]).await; - let store = Arc::new(open(&dir).await); - let (addr, content) = addressed("written-then-pruned"); - - // The gate is held from its own thread, so nothing holds a blocking guard across - // an await, and it is released through a channel when the test is ready. - let gate = store.files.test_put_gate(); - let (held_tx, held_rx) = std::sync::mpsc::channel::<()>(); - let (release_tx, release_rx) = std::sync::mpsc::channel::<()>(); - let holder = std::thread::spawn(move || { - let _parked = gate.write(); - held_tx.send(()).ok(); - release_rx.recv().ok(); - }); - held_rx.recv().expect("the gate is held"); - - // The state under test, built directly rather than by racing a real put: a write - // that announced itself, whose file half is parked mid-publish, and whose caller - // is gone. Driving it through `put` and aborting would be a race about which half - // had started, and a test that sometimes sets up a different state than it claims - // is worse than no test. - let legacy = store.legacy().expect("legacy"); - legacy.announce(&addr); - let publishing = { - let files = Arc::clone(&store.files); + let store = Arc::new(reopen(&dir).await); + let content = b"a chunk the pruner has decided to drop".to_vec(); + let addr = crate::client::compute_address(&content); + + // Park the put in the window that matters: it has taken the key's lane and has NOT + // yet registered itself, so a delete's wait for in-flight writes would see nothing. + // Parking it later, inside its closure, cannot show the lane doing anything: the + // delete would block on the wait instead and the test would pass either way. + let gate = store.test_pre_registration_gate(); + let held = gate.write().await; + let writing = { + let store = Arc::clone(&store); let content = content.clone(); - tokio::spawn(async move { files.put(&addr, &content).await }) + tokio::spawn(async move { store.put(&addr, &content).await }) }; - // Until the publish is genuinely in flight and parked at the gate. - while store.files.tasks_in_flight() == 0 { + // Waited for, not slept at. A sleep makes the staging a guess, and on a loaded + // machine the guess is wrong and the test fails for the wrong reason. + let deadline = std::time::Instant::now() + Duration::from_secs(30); + while store.test_reached_pre_registration() == 0 { + assert!( + std::time::Instant::now() < deadline, + "the put never reached the gate" + ); tokio::time::sleep(Duration::from_millis(5)).await; } - publishing.abort(); - let _ = publishing.await; - // The delete has to wait the parked half out rather than racing it. + // The delete must not get past the lane while that put holds it. Without the lane + // on `put` this finishes immediately, which is the regression. let deleting = { let store = Arc::clone(&store); tokio::spawn(async move { store.delete(&addr).await }) }; - tokio::time::sleep(Duration::from_millis(100)).await; + tokio::time::sleep(Duration::from_millis(200)).await; assert!( !deleting.is_finished(), - "the delete must wait for the write it would otherwise race" + "the delete finished while a put for the same key was still under way" ); - release_tx.send(()).ok(); - holder.join().ok(); - deleting.await.expect("join").expect("delete"); + // Released: the put completes, then the delete runs. The delete is the later + // decision, so the chunk must be gone. + drop(held); + let _ = writing.await.expect("the put task must not panic"); + let _ = deleting.await.expect("the delete task must not panic"); + store.wait_idle().await; - // Whichever half landed, the key is gone and stays gone. - store.files.wait_idle().await; assert!( - !store.exists(&addr).unwrap_or(true), - "a write that landed after the delete would resurrect a pruned chunk" + !store.is_indexed(&addr), + "the put landed after the delete and put {} back", + hex::encode(addr) + ); + assert!( + !store.chunk_path(&addr).exists(), + "and left its file behind" ); - assert!(!store.legacy_only_keys().contains(&addr)); } - /// A delete outlasts a file write that no journal knows about. + /// A delete outlasts a write nobody waited for. + /// + /// A write's blocking half outlives the future that started it, deliberately, so the + /// work is never left half done. That means a cancelled put can still be queued when a + /// delete arrives, and if the delete does not wait for it the write lands afterwards + /// and puts back a chunk the node had decided to prune. The key is then in a store that + /// no longer claims it, which is what the next verification has to clean up. /// - /// The journal is kept by writes that touch both stores. The copier and the repair - /// path write only the file, so a delete that consulted the journal to decide whether - /// to wait would not wait for either of them, and whichever landed afterwards would - /// put back a chunk the node had decided to prune. What is writing a key is the file - /// store's own question to answer. + /// This ordering had a regression test before the migration facade was deleted, and the + /// test went with the facade even though the requirement did not. #[tokio::test] - async fn a_delete_outlasts_a_file_write_with_no_journal_entry() { + // The gate is held across awaits deliberately: holding it is what parks the put, which + // is the state the delete has to be ordered against. + #[allow(clippy::await_holding_lock)] + async fn a_delete_outlasts_a_write_nobody_waited_for() { let dir = TempDir::new().expect("temp dir"); - seed_legacy(&dir, &["neighbour"]).await; - let store = Arc::new(open(&dir).await); - let (addr, content) = addressed("copied-then-pruned"); - - let gate = store.files.test_put_gate(); - let (held_tx, held_rx) = std::sync::mpsc::channel::<()>(); - let (release_tx, release_rx) = std::sync::mpsc::channel::<()>(); - let holder = std::thread::spawn(move || { - let _parked = gate.write(); - held_tx.send(()).ok(); - release_rx.recv().ok(); - }); - held_rx.recv().expect("the gate is held"); - - // Deliberately no journal entry: this is the copier's shape, not a dual write. - let publishing = { - let files = Arc::clone(&store.files); + let store = Arc::new(reopen(&dir).await); + let content = b"a chunk that is about to be pruned".to_vec(); + let addr = crate::client::compute_address(&content); + + // Park the put inside its closure, then drop the future waiting on it. + let gate = store.test_put_gate(); + let held = gate.write(); + let put = { + let store = Arc::clone(&store); let content = content.clone(); - tokio::spawn(async move { files.put(&addr, &content).await }) + tokio::spawn(async move { store.put(&addr, &content).await }) }; - while store.files.tasks_in_flight() == 0 { + let deadline = std::time::Instant::now() + Duration::from_secs(30); + while store.tasks_in_flight() == 0 { + assert!( + std::time::Instant::now() < deadline, + "the put never reached the closure" + ); tokio::time::sleep(Duration::from_millis(5)).await; } - publishing.abort(); - let _ = publishing.await; - assert!( - !store - .legacy() - .is_some_and(|l| l.pending.read().contains_key(&addr)), - "this is the case the journal does not cover, so it must be empty" - ); + put.abort(); + let _ = put.await; + // The delete must not finish ahead of that write. Released after the delete has + // had time to be waiting, so if it does not wait, it wins the race and the test + // catches it. let deleting = { let store = Arc::clone(&store); tokio::spawn(async move { store.delete(&addr).await }) }; tokio::time::sleep(Duration::from_millis(100)).await; + drop(held); + let _ = deleting + .await + .expect("the delete task itself must not fail"); + store.wait_idle().await; + assert!( - !deleting.is_finished(), - "the delete must wait for a write the journal never knew about" + !store.is_indexed(&addr), + "the write landed after the delete and put {} back", + hex::encode(addr) ); - - release_tx.send(()).ok(); - holder.join().ok(); - deleting.await.expect("join").expect("delete"); - - store.files.wait_idle().await; assert!( - !store.exists(&addr).unwrap_or(true), - "a write that landed after the delete would resurrect a pruned chunk" + !store.chunk_path(&addr).exists(), + "and left its file on disk" ); } - /// A single node can still be told to keep both stores. + /// A put whose caller goes away does not admit a key on bytes nothing has read. + /// + /// The blocking half of a put outlives the future that started it, deliberately, so + /// the work is never left half done. That makes anything it writes to memory a claim + /// the node keeps whether or not the caller is still there to finish checking it. + /// + /// For a chunk this call published the claim is earned: the bytes were hashed against + /// their own name on the way in. For a name that was already taken it is not. The + /// check that decides whether those bytes are good runs after the await, and a dropped + /// future skips it, so admitting the key in the closure claims a chunk nobody read. + /// + /// Staged with a fifo, which is the sharpest case and a real one: the startup scan + /// refuses non-regular entries by design, so this is a key the store has already + /// decided it must not claim, walked in through the back door. + #[cfg(unix)] #[tokio::test] - async fn retirement_is_refused_when_the_switch_is_turned_off() { + // The gate is held across an await deliberately: holding it is what parks the put + // inside its closure, which is the state under test. Dropping it before awaiting would + // let the put finish and there would be nothing to cancel. + #[allow(clippy::await_holding_lock)] + async fn a_cancelled_put_does_not_admit_a_key_whose_bytes_were_never_read() { let dir = TempDir::new().expect("temp dir"); - seed_legacy(&dir, &["off"]).await; - let mut config = test_config(&dir); - config.migration.retire_legacy = false; - let store = ChunkStore::new(config).await.expect("open store"); - store.commit_to_files().expect("commit"); - assert!(store - .retirement_blocker(|_| false) - .expect("blocked") - .contains("retirement is disabled")); + let store = Arc::new(reopen(&dir).await); + + // A name a real chunk would use, wearing something that is not a chunk. + let content = b"the bytes that belong under this name".to_vec(); + let addr = crate::client::compute_address(&content); + let shard = dir + .path() + .join(CHUNKS_DIR_NAME) + .join(format!("{:02x}", addr[31])); + std::fs::create_dir_all(&shard).expect("mkdir"); + let path = shard.join(hex::encode(addr)); + let name = std::ffi::CString::new(path.as_os_str().as_encoded_bytes()) + .expect("a path with no interior nul"); + // SAFETY: `name` is a valid NUL-terminated C string that outlives the call, and the + // mode is a constant. `mkfifo` reads the pointer and returns; nothing is retained. + #[allow(clippy::undocumented_unsafe_blocks, unsafe_code)] + let made = unsafe { libc::mkfifo(name.as_ptr(), 0o644) }; + assert_eq!(made, 0, "could not make the fifo this test needs"); + + // Hold the gate so the put parks inside the closure, then drop the future while it + // is parked. That is a caller going away mid-put, which is what a cancelled + // request, a client disconnect or a shutdown all look like from in here. + let gate = store.test_put_gate(); + let held = gate.write(); + let put = { + let store = Arc::clone(&store); + let content = content.clone(); + tokio::spawn(async move { store.put(&addr, &content).await }) + }; + // Waited for rather than slept at. A sleep proves nothing: if the put had not + // reached the gated closure yet, aborting would cancel it before it ever got + // there and the test would pass having staged nothing. + let deadline = std::time::Instant::now() + Duration::from_secs(30); + while store.tasks_in_flight() == 0 { + assert!( + std::time::Instant::now() < deadline, + "the put never reached the closure, so there was nothing to cancel" + ); + tokio::time::sleep(Duration::from_millis(5)).await; + } + put.abort(); + let _ = put.await; + drop(held); + store.wait_idle().await; + + assert!( + !store.is_indexed(&addr), + "a cancelled put admitted {} on bytes nothing read; the fifo under that name \ + would then be advertised, committed to, and audited against", + hex::encode(addr) + ); + assert!( + !store.exists(&addr).unwrap_or(true), + "and the node must not claim it either" + ); } + /// A marker temporary left in the node root is swept, and nothing else is. + /// + /// The migration marker is written next to itself in the root, which no sweep looked + /// at, so a crash between its write and its rename left one there for the life of the + /// node. Small, but nothing was ever going to remove it. + /// + /// The second half is the point: this runs over a directory holding a node's data, so + /// it has to take only the exact shape this module writes and leave everything else + /// where it is. #[tokio::test] - async fn retiring_removes_the_legacy_environment_and_frees_its_space() { + async fn a_leftover_marker_temporary_is_swept_and_its_neighbours_are_not() { let dir = TempDir::new().expect("temp dir"); - let keys = seed_legacy(&dir, &["f1", "f2", "f3"]).await; - let mut config = test_config(&dir); - config.migration.retire_legacy = true; - let store = ChunkStore::new(config).await.expect("open"); - - store - .copy_batch(&keys, 0, 0, &never_cancelled()) - .await - .expect("copy"); - open_the_retirement_gate(&store); + let root = dir.path(); + let leftover = root.join(format!("{TEMP_PREFIX}1234.abcdef01.marker")); + std::fs::write(&leftover, b"an interrupted marker write").expect("plant"); + + // Things that must survive: the marker itself, a chunk-shaped temp that belongs to + // the chunk tree's own sweep, and anything an operator put there. + let keep = [ + root.join("migration-state.json"), + root.join(format!("{TEMP_PREFIX}1234.abcdef01.chunk")), + root.join("notes.txt"), + // Prefix and suffix alone would take these. The pid and the nonce are checked + // because this runs over a directory holding a node's data. + root.join(format!("{TEMP_PREFIX}operator-notes.marker")), + root.join(format!("{TEMP_PREFIX}1234.nothex01.marker")), + root.join(format!("{TEMP_PREFIX}1234.abcdef0.marker")), + root.join(format!("{TEMP_PREFIX}1234.abcdef01.extra.marker")), + ]; + for path in &keep { + std::fs::write(path, b"keep me").expect("plant"); + } - let proof = store - .verify_before_retire(0, &never_cancelled()) - .await - .expect("verify"); - assert!(proof.is_clean()); - assert_eq!(proof.checked, 3); + let store = reopen(&dir).await; + drop(store); - let freed = store - .retire_legacy(&proof, &|_: &XorName| false, &approved_shed(&store)) - .await - .expect("retire"); - assert!(freed > 0, "retirement must report the space it returned"); assert!( - !dir.path().join(LEGACY_ENV_DIR).exists(), - "the legacy environment must actually be removed" + !leftover.exists(), + "the leftover marker temporary is still in the node root" ); - assert!(!store.has_legacy()); - assert_eq!(store.migration_phase(), MigrationPhase::FilesOnly); - - // Everything is still readable, from files alone. - assert_eq!(store.current_chunks().expect("count"), 3); - for key in &keys { - assert!(store.get(key).await.expect("get").is_some()); + for path in &keep { + assert!( + path.exists(), + "{} was swept and should not have been", + path.display() + ); } } #[tokio::test] - async fn a_reader_holding_the_legacy_handle_defers_retirement_without_hiding_chunks() { - let dir = TempDir::new().expect("temp dir"); - let keys = seed_legacy(&dir, &["busy"]).await; - let mut config = test_config(&dir); - config.migration.retire_legacy = true; - let store = Arc::new(ChunkStore::new(config).await.expect("open")); - store - .copy_batch(&keys, 0, 0, &never_cancelled()) - .await - .expect("copy"); - open_the_retirement_gate(&store); - - // Stand in for a read that is still holding the legacy handle. Retirement must - // defer rather than unmap underneath it, and the chunk must stay readable - // throughout: a retirement attempt that briefly hid the legacy store would make a - // node answer "not found" for a chunk it holds. - let squatter = store.legacy().expect("a legacy handle"); - let proof = store - .verify_before_retire(0, &never_cancelled()) - .await - .expect("verify"); - let err = store - .retire_legacy(&proof, &|_: &XorName| false, &approved_shed(&store)) - .await - .expect_err("must defer while the handle is held"); - assert!(format!("{err}").contains("deferred"), "{err}"); + async fn a_corrupt_chunk_is_removed_so_replication_can_repair_it() { + let (store, _dir) = test_store().await; + let (addr, content) = addressed("corrupt"); + store.put(&addr, &content).await.expect("put"); + std::fs::write(store.chunk_path(&addr), b"tampered").expect("tamper"); - assert!(store.has_legacy(), "the store must keep its legacy handle"); - let key = keys.first().copied().expect("a key"); + let err = store.get(&addr).await.expect_err("verification must fail"); + assert!(format!("{err}").contains("verification failed"), "{err}"); + + assert!(!store.chunk_path(&addr).exists(), "corrupt file must go"); + assert!(!store.exists(&addr).expect("exists")); assert!( - store.get(&key).await.expect("get").is_some(), - "the chunk must stay readable across a deferred retirement" + !store.all_keys().await.expect("all_keys").contains(&addr), + "a corrupt chunk must stop being advertised" ); - - // Once the reader lets go, the next attempt succeeds. - drop(squatter); - store - .retire_legacy(&proof, &|_: &XorName| false, &approved_shed(&store)) - .await - .expect("retire"); - assert!(!store.has_legacy()); + assert_eq!(store.stats().verification_failures, 1); } #[tokio::test] - async fn retirement_needs_a_clean_verification_report() { - let dir = TempDir::new().expect("temp dir"); - // Left uncopied on purpose, so this node is about to give the chunk up and the - // answerability veto has something to fire on. - seed_legacy(&dir, &["v1"]).await; - let mut config = test_config(&dir); - config.migration.retire_legacy = true; - let store = ChunkStore::new(config).await.expect("open"); - - // A report cannot be fabricated: every field is private and the only source is - // the verification pass. The one available here is the default, which never ran. - let absent = VerifyReport::default(); - let err = store - .retire_legacy(&absent, &|_: &XorName| false, &approved_shed(&store)) - .await - .expect_err("must refuse a report that never ran"); - assert!(format!("{err}").contains("unrepairable"), "{err}"); - assert!(store.has_legacy(), "the legacy environment must survive"); - - // And a real, clean report is still refused while any gate is unmet, because - // retirement rechecks them all itself rather than trusting its caller. - open_the_retirement_gate(&store); - let proof = store - .verify_before_retire(0, &never_cancelled()) - .await - .expect("verify"); - assert!(proof.is_clean()); - let err = store - .retire_legacy(&proof, &|_: &XorName| true, &approved_shed(&store)) - .await - .expect_err("must refuse while a chunk is still answerable"); - assert!(format!("{err}").contains("still answerable"), "{err}"); - assert!(store.has_legacy()); + async fn a_file_removed_underneath_the_store_drops_out_of_the_index() { + let (store, _dir) = test_store().await; + let (addr, content) = addressed("vanished"); + store.put(&addr, &content).await.expect("put"); + + std::fs::remove_file(store.chunk_path(&addr)).expect("remove behind our back"); + + assert!(store.get(&addr).await.expect("get").is_none()); + assert!(!store.exists(&addr).expect("exists")); + assert_eq!(store.current_chunks().expect("count"), 0); } #[tokio::test] - async fn verification_repairs_a_file_that_rotted_before_retirement() { - let dir = TempDir::new().expect("temp dir"); - let keys = seed_legacy(&dir, &["w1", "w2"]).await; - let mut config = test_config(&dir); - config.migration.retire_legacy = true; - let store = ChunkStore::new(config).await.expect("open"); - store - .copy_batch(&keys, 0, 0, &never_cancelled()) - .await - .expect("copy"); + async fn interrupted_writes_are_swept_at_startup() { + let (store, dir) = test_store().await; + let (addr, content) = addressed("sweep"); + store.put(&addr, &content).await.expect("put"); + let shard = store + .chunk_path(&addr) + .parent() + .expect("shard") + .to_path_buf(); + drop(store); - // A filename is not proof the bytes behind it are good. Corrupt one, exactly as - // a truncated write or a bad sector would, then prove retirement repairs it - // rather than deleting the only intact copy. - let victim = keys.first().copied().expect("a key"); - let path = dir + let orphan = shard.join(format!("{TEMP_PREFIX}999.7")); + std::fs::write(&orphan, b"half a chunk").expect("write orphan"); + let stray_root = dir .path() - .join(crate::storage::file_store::CHUNKS_DIR_NAME) - .join(format!("{:02x}", victim.last().copied().unwrap_or(0))) - .join(hex::encode(victim)); - std::fs::write(&path, b"rotted").expect("corrupt the file"); - open_the_retirement_gate(&store); - - let proof = store - .verify_before_retire(0, &never_cancelled()) - .await - .expect("verify"); - assert_eq!(proof.repaired(), 1); - assert_eq!(proof.unrepairable(), 0); - assert!(proof.is_clean()); + .join(CHUNKS_DIR_NAME) + .join(format!("{TEMP_PREFIX}999.8")); + std::fs::write(&stray_root, b"half a marker").expect("write stray"); - store - .retire_legacy(&proof, &|_: &XorName| false, &approved_shed(&store)) - .await - .expect("retire"); - assert_eq!( - store.get(&victim).await.expect("get").expect("present"), - addressed("w1").1 - ); + let reopened = reopen(&dir).await; + assert!(!orphan.exists(), "an interrupted write must not survive"); + assert!(!stray_root.exists(), "nor one at the store root"); + assert_eq!(reopened.current_chunks().expect("count"), 1); } #[tokio::test] - async fn a_corrupt_file_is_served_from_the_legacy_copy_and_requeued() { - let dir = TempDir::new().expect("temp dir"); - let keys = seed_legacy(&dir, &["s1"]).await; - let store = open(&dir).await; - store - .copy_batch(&keys, 0, 0, &never_cancelled()) - .await - .expect("copy"); - assert!(store.legacy_only_keys().is_empty()); + async fn concurrent_writers_of_one_address_store_it_exactly_once() { + let (store, _dir) = test_store().await; + let store = Arc::new(store); + let (addr, content) = addressed("racing"); - let victim = keys.first().copied().expect("a key"); - let path = dir - .path() - .join(crate::storage::file_store::CHUNKS_DIR_NAME) - .join(format!("{:02x}", victim.last().copied().unwrap_or(0))) - .join(hex::encode(victim)); - std::fs::write(&path, b"rotted").expect("corrupt the file"); + let mut tasks = Vec::new(); + for _ in 0..16 { + let store = Arc::clone(&store); + let content = content.clone(); + tasks.push(tokio::spawn( + async move { store.put(&addr, &content).await }, + )); + } - // The file store removes the bad file and stops advertising it; the facade must - // still find the intact copy rather than reporting a failure. + let mut new_count = 0; + for task in tasks { + if task.await.expect("join").expect("put") { + new_count += 1; + } + } + assert_eq!(new_count, 1, "exactly one writer may report a new chunk"); + assert_eq!(store.current_chunks().expect("count"), 1); assert_eq!( - store.get(&victim).await.expect("get").expect("present"), - addressed("s1").1 - ); - assert!( - store.legacy_only_keys().contains(&victim), - "the key must go back on the copier's list" + store.get(&addr).await.expect("get").expect("present"), + content ); } #[tokio::test] - async fn a_marker_claiming_more_than_the_file_store_holds_restarts_the_copy() { - let dir = TempDir::new().expect("temp dir"); - let keys = seed_legacy(&dir, &["p1", "p2", "p3"]).await; - let store = open(&dir).await; - store - .copy_batch(&keys, 0, 0, &never_cancelled()) - .await - .expect("copy"); - store.commit_to_files().expect("commit"); - assert_eq!(store.migration_state().kept_key_count, 3); - store.wait_idle().await; + async fn names_that_are_not_lowercase_hex_are_ignored_by_the_scan() { + let (store, dir) = test_store().await; + let (addr, content) = addressed("scan"); + store.put(&addr, &content).await.expect("put"); + let shard = store + .chunk_path(&addr) + .parent() + .expect("shard") + .to_path_buf(); drop(store); - // Someone clears the chunk directory to reclaim space, keeping chunks.mdb. - // Trusting the marker here would skip the copier, the shed rules and their rank - // checks on the way to deleting the legacy environment. - let chunks = dir.path().join(crate::storage::file_store::CHUNKS_DIR_NAME); - for key in &keys { - let path = chunks - .join(format!("{:02x}", key.last().copied().unwrap_or(0))) - .join(hex::encode(key)); - std::fs::remove_file(path).expect("clear the file store"); - } + // Uppercase is deliberately rejected: on a case-folding filesystem accepting it + // would let one file answer to two index entries. + let upper = shard.join(hex::encode_upper(addressed("upper").0)); + std::fs::write(&upper, b"x").expect("write upper"); + std::fs::write(shard.join("not-a-chunk"), b"x").expect("write junk"); + std::fs::write(shard.join("deadbeef"), b"x").expect("write short"); - let store = open(&dir).await; - assert_eq!( - store.migration_phase(), - MigrationPhase::Bridging, - "the filesystem must win over the marker" - ); - assert_eq!(store.legacy_only_keys().len(), 3); + let reopened = reopen(&dir).await; + assert_eq!(reopened.current_chunks().expect("count"), 1); } - /// V2-1232. Once committed, the copier moves only the keys the rank check refuses to - /// shed, so a chunk that is found in the legacy environment after the commitment, - /// and that this node never agreed to give up, is never copied, never provable - /// elsewhere, and keeps the environment from ever being retired. Free disk does not - /// help and neither does a restart. Reopening has to notice and reopen the bridge. #[tokio::test] - async fn a_node_that_committed_with_nothing_to_shed_resumes_the_copy_for_chunks_only_it_holds() - { - let dir = TempDir::new().expect("temp dir"); - let keys = seed_legacy(&dir, &["c1", "c2"]).await; - let store = open(&dir).await; - store - .copy_batch(&keys, 0, 0, &never_cancelled()) - .await - .expect("copy"); - store.commit_to_files().expect("commit"); - store.note_commitment_rebuilt(); - let state = store.migration_state(); - assert_eq!(state.shed_key_count, 0, "nothing had to be shed"); - assert!(state.committed_at_unix.is_some()); - assert_eq!(state.rebuilds_since_commit, 1); - store.wait_idle().await; + async fn a_chunk_filed_in_the_wrong_shard_is_not_indexed() { + let (store, dir) = test_store().await; + let (addr, content) = addressed("misfiled"); + store.put(&addr, &content).await.expect("put"); drop(store); - // A chunk in the legacy environment that the file store has no record of, found - // after the node committed. The file store is untouched, so the kept-count rule - // has nothing to say about this. - let late = seed_legacy(&dir, &["c3"]).await; - - let store = open(&dir).await; - let state = store.migration_state(); - assert_eq!( - state.phase, - MigrationPhase::Bridging, - "a chunk the node never agreed to shed must reopen the bridge" - ); - assert!( - state.committed_at_unix.is_none(), - "the retirement clock must not carry over from the abandoned commitment" - ); - assert_eq!(state.rebuilds_since_commit, 0); - assert_eq!(store.legacy_only_keys(), late); + // Move it one shard over: the read path would never find it there, so indexing + // it would make the store advertise a key it cannot serve. + let correct = dir + .path() + .join(CHUNKS_DIR_NAME) + .join(shard_name(&addr)) + .join(hex::encode(addr)); + let wrong_shard_index = (shard_index(&addr) + 1) % SHARD_COUNT; + let wrong_dir = dir + .path() + .join(CHUNKS_DIR_NAME) + .join(format!("{wrong_shard_index:02x}")); + std::fs::create_dir_all(&wrong_dir).expect("mkdir"); + std::fs::rename(&correct, wrong_dir.join(hex::encode(addr))).expect("misfile"); - // And the copier, which the phase change lets run again, moves it. - store - .copy_batch(&late, 0, 0, &never_cancelled()) - .await - .expect("copy"); - assert!(store.legacy_only_keys().is_empty()); - assert_eq!( - store - .get(late.first().expect("a key")) - .await - .expect("get") - .expect("present"), - addressed("c3").1 - ); + let reopened = reopen(&dir).await; + assert_eq!(reopened.current_chunks().expect("count"), 0); + assert!(!reopened.exists(&addr).expect("exists")); } - /// The counterpart of the test above, and it matters as much. A node that shed keeps - /// exactly the keys it is giving up in the legacy environment until they stop being - /// answerable, so a non-empty legacy-only set is its normal state. Sending it back - /// to the bridge on every restart would reset its retention clock each time, and a - /// node restarted regularly would never retire. #[tokio::test] - async fn a_node_that_shed_stays_committed_across_a_restart() { - let dir = TempDir::new().expect("temp dir"); - let keys = seed_legacy(&dir, &["k1", "k2"]).await; - let store = open(&dir).await; - store - .copy_batch(&keys[..1], 0, 0, &never_cancelled()) - .await - .expect("copy"); - store.commit_to_files().expect("commit"); - store.note_commitment_rebuilt(); - let before = store.migration_state(); - assert_eq!(before.shed_key_count, 1); - assert_eq!(store.legacy_only_keys().len(), 1); - store.wait_idle().await; + async fn the_layout_marker_is_written_once_and_checked_on_reopen() { + let (store, dir) = test_store().await; drop(store); - let store = open(&dir).await; - let after = store.migration_state(); - assert_eq!( - after.phase, - MigrationPhase::Committed, - "the keys it is giving up are not a reason to reopen the bridge" - ); - assert_eq!(after.committed_at_unix, before.committed_at_unix); - assert_eq!(after.rebuilds_since_commit, before.rebuilds_since_commit); - assert_eq!(store.legacy_only_keys().len(), 1); + let marker = dir.path().join(CHUNKS_DIR_NAME).join(LAYOUT_FILE_NAME); + let layout: StoreLayout = + serde_json::from_slice(&std::fs::read(&marker).expect("read marker")) + .expect("parse marker"); + assert_eq!(layout, StoreLayout::default()); + + // A store written by a future build must be refused, not misread. + let future = StoreLayout { + schema: LAYOUT_SCHEMA + 1, + ..StoreLayout::default() + }; + std::fs::write(&marker, serde_json::to_vec(&future).expect("encode")).expect("write"); + let err = ChunkStore::new(ChunkStoreConfig { + root_dir: dir.path().to_path_buf(), + verify_on_read: true, + disk_reserve: 0, + }) + .await + .expect_err("must refuse a newer layout"); + assert!(format!("{err}").contains("newer than this build"), "{err}"); } #[tokio::test] - async fn a_file_that_vanished_mid_verification_is_requeued_not_republished() { - let dir = TempDir::new().expect("temp dir"); - let keys = seed_legacy(&dir, &["gone"]).await; - let mut config = test_config(&dir); - config.migration.retire_legacy = true; - let store = ChunkStore::new(config).await.expect("open"); - store - .copy_batch(&keys, 0, 0, &never_cancelled()) - .await - .expect("copy"); - assert!(store.legacy_only_keys().is_empty()); - - // Remove the file without telling the store, which is what the pruner's own - // delete looks like if it lands mid-pass. Republishing from the legacy copy here - // would resurrect a chunk the node had deliberately deleted, so the key goes back - // on the copier's list instead and retirement is refused. - let key = keys.first().copied().expect("a key"); - let path = dir - .path() - .join(crate::storage::file_store::CHUNKS_DIR_NAME) - .join(format!("{:02x}", key.last().copied().unwrap_or(0))) - .join(hex::encode(key)); - std::fs::remove_file(&path).expect("remove behind the store's back"); - - let proof = store - .verify_before_retire(0, &never_cancelled()) - .await - .expect("verify"); - assert!( - proof.unrepairable >= 1, - "the vanished file must be counted against the proof" - ); - assert!(!proof.is_clean()); - assert!(!path.exists(), "the pass must not republish it"); - assert!(store.legacy_only_keys().contains(&key)); - assert!(store - .retire_legacy(&proof, &|_: &XorName| false, &approved_shed(&store)) - .await - .is_err()); - assert!(store.has_legacy()); + async fn an_unknown_shard_scheme_is_refused() { + let (store, dir) = test_store().await; + drop(store); + let marker = dir.path().join(CHUNKS_DIR_NAME).join(LAYOUT_FILE_NAME); + let other = StoreLayout { + scheme: "prefix-hex".to_string(), + ..StoreLayout::default() + }; + std::fs::write(&marker, serde_json::to_vec(&other).expect("encode")).expect("write"); + let err = ChunkStore::new(ChunkStoreConfig { + root_dir: dir.path().to_path_buf(), + verify_on_read: true, + disk_reserve: 0, + }) + .await + .expect_err("must refuse an unknown scheme"); + assert!(format!("{err}").contains("shard scheme"), "{err}"); } #[tokio::test] - async fn a_cancelled_shutdown_stops_the_copier_and_refuses_to_pass_verification() { + async fn writes_are_refused_when_the_disk_reserve_cannot_be_met() { let dir = TempDir::new().expect("temp dir"); - let keys = seed_legacy(&dir, &["s1", "s2", "s3"]).await; - let store = open(&dir).await; - - // Shutdown must not have to wait out a pass that can run for hours, and a pass - // that stopped early is not evidence of anything. - let cancelled = CancellationToken::new(); - cancelled.cancel(); - - let report = store - .copy_batch(&keys, 0, 0, &cancelled) - .await - .expect("copy"); - assert_eq!(report.copied, 0, "the copier must stop immediately"); - assert_eq!(store.legacy_only_keys().len(), 3); + let store = ChunkStore::new(ChunkStoreConfig { + root_dir: dir.path().to_path_buf(), + verify_on_read: true, + disk_reserve: u64::MAX / 2, + }) + .await + .expect("open store"); - let proof = store - .verify_before_retire(0, &cancelled) - .await - .expect("verify"); + let (addr, content) = addressed("full"); + let err = store.put(&addr, &content).await.expect_err("must refuse"); assert!( - !proof.is_clean(), - "an interrupted verification must never read as a pass" + format!("{err}").contains("Insufficient disk space"), + "{err}" ); + assert!(store.check_capacity().is_err()); } #[tokio::test] - async fn the_migration_marker_survives_a_restart() { + async fn capacity_is_size_aware() { + // Wide enough that a test running alongside this one cannot move the answer. + const MARGIN: u64 = 512 * 1024 * 1024; + let dir = TempDir::new().expect("temp dir"); - let keys = seed_legacy(&dir, &["m1", "m2"]).await; - let store = open(&dir).await; - store - .copy_batch(&keys[..1], 0, 0, &never_cancelled()) - .await - .expect("copy"); - store.commit_to_files().expect("commit"); - store.note_commitment_rebuilt(); - let first_start = store.migration_state().first_start_unix; - store.wait_idle().await; - drop(store); + let available = fs2::available_space(dir.path()).expect("free space"); + // A reserve that leaves room for a small write but not a huge one. This is the + // whole reason the predicate takes a size: free bytes alone stopped being a + // sufficient answer once chunks became files. + let store = ChunkStore::new(ChunkStoreConfig { + root_dir: dir.path().to_path_buf(), + verify_on_read: true, + disk_reserve: available.saturating_sub(MARGIN), + }) + .await + .expect("open store"); + + assert!(store.check_capacity_for(1024).is_ok()); + assert!(store.check_capacity_for(4 * MARGIN).is_err()); + } - let store = open(&dir).await; - let state = store.migration_state(); - assert_eq!(state.phase, MigrationPhase::Committed); - assert_eq!(state.shed_key_count, 1); + #[test] + fn suffix_shards_stay_uniform_for_a_close_group_of_keys() { + // The real distribution: a node holds keys it is closest to, so they share a + // long leading prefix with its own ID. Sharding on that prefix collapses to one + // directory. The trailing byte is untouched by close-group membership. + let mut prefix_dirs = HashSet::new(); + let mut suffix_dirs = HashSet::new(); + for i in 0u32..4096 { + let mut key = [0u8; XORNAME_LEN]; + // 20 shared leading bits, as a ~1M-node network would impose. + let tail = crate::client::compute_address(&i.to_le_bytes()); + key.copy_from_slice(&tail); + if let Some(b) = key.first_mut() { + *b = 0xab; + } + if let Some(b) = key.get_mut(1) { + *b = 0xcd; + } + if let Some(b) = key.get_mut(2) { + *b &= 0x0f; + } + prefix_dirs.insert(key.first().copied().unwrap_or(0)); + suffix_dirs.insert(shard_index(&key)); + } assert_eq!( - state.first_start_unix, first_start, - "a restart must not restart the shed hold" + prefix_dirs.len(), + 1, + "prefix sharding collapses for a node's own holdings" ); assert!( - state.committed_at_unix.is_some(), - "nor the retirement clock" + suffix_dirs.len() > 250, + "suffix sharding must stay uniform, got {} of 256 directories", + suffix_dirs.len() ); } + #[test] + fn no_chunk_filename_can_spell_a_reserved_windows_device_name() { + // Hex has no `n`, `u`, `x`, `p`, `r`, `l`, `t`, `o` or `s`, so `CON`, `NUL`, + // `AUX`, `PRN`, `COM1` and `LPT1` are all unspellable at any length. This is why + // the encoding is hex and not base32 or base64url. + for reserved in ["con", "prn", "aux", "nul", "com1", "com9", "lpt1", "lpt9"] { + assert!( + !is_lower_hex(reserved), + "{reserved} must not be a valid chunk or shard name" + ); + } + } + + #[test] + fn only_full_length_lowercase_hex_decodes_to_an_address() { + // 0xab so the hex form actually contains letters, which is where case matters. + assert!(decode_chunk_name(&hex::encode([0xabu8; XORNAME_LEN])).is_some()); + assert!(decode_chunk_name(&hex::encode_upper([0xabu8; XORNAME_LEN])).is_none()); + assert!(decode_chunk_name("deadbeef").is_none()); + assert!(decode_chunk_name("").is_none()); + assert!(decode_chunk_name(&"g".repeat(CHUNK_NAME_LEN)).is_none()); + } + #[tokio::test] - async fn a_marker_that_disagrees_with_the_filesystem_loses() { - let dir = TempDir::new().expect("temp dir"); - seed_legacy(&dir, &["x1"]).await; - let store = open(&dir).await; - store.force_migration_state(|s| s.phase = MigrationPhase::FilesOnly); - store.migration_state().save(dir.path()).expect("save"); - store.wait_idle().await; - drop(store); + async fn repair_replaces_bad_bytes_without_the_file_ever_being_absent() { + let (store, _dir) = test_store().await; + let (addr, content) = addressed("repairable"); + store.put(&addr, &content).await.expect("put"); + let path = store.chunk_path(&addr); + + std::fs::write(&path, b"rotted").expect("corrupt"); + store.repair(&addr, &content).await.expect("repair"); - // The marker claims the migration is done, but `chunks.mdb` is right there. The - // filesystem is the authority. - let store = open(&dir).await; - assert_eq!(store.migration_phase(), MigrationPhase::Bridging); - assert!(store.has_legacy()); + assert_eq!( + store.get(&addr).await.expect("get").expect("present"), + content + ); + assert!(store.exists(&addr).expect("exists")); } - /// A legacy record whose bytes do not hash to its key is removed, not passed around. - /// - /// Leaving it in the environment while dropping it from the key set puts it in - /// neither view, and the pre-retirement pass reads a key in neither view as one to - /// protect and puts it straight back. The next copier pass drops it again. One rotted - /// record would keep this node, and every node sharing its disk, from ever reclaiming - /// space. #[tokio::test] - async fn a_legacy_chunk_that_does_not_match_its_address_is_removed_not_recycled() { - let dir = TempDir::new().expect("temp dir"); - let (addr, _) = addressed("bad"); - let other = addressed("other").1; - { - let lmdb = LmdbStorage::new(LmdbStorageConfig { - root_dir: dir.path().to_path_buf(), - verify_on_read: false, - max_map_size: 0, - disk_reserve: 0, - }) - .await - .expect("open legacy"); - // Under a key it does not hash to: what a record that rotted in place looks - // like, and the one shape the ordinary path refuses to create. - lmdb.put_unchecked(&addr, &other).await.expect("put"); - lmdb.wait_idle().await; - } - let store = open(&dir).await; - let keys = store.legacy_only_keys(); - assert_eq!(keys, vec![addr]); + async fn a_repair_with_the_wrong_bytes_is_refused_and_changes_nothing() { + let (store, _dir) = test_store().await; + let (addr, content) = addressed("guarded"); + store.put(&addr, &content).await.expect("put"); + let path = store.chunk_path(&addr); - let report = store - .copy_batch(&keys, 0, 0, &never_cancelled()) - .await - .expect("copy"); - assert_eq!(report.copied, 0); - assert_eq!(report.unusable, 1); - assert!(store.legacy_only_keys().is_empty()); - - // And it is gone from the environment, so the pass below cannot find it and put - // it back. That is the loop this is about. - let proof = store - .verify_before_retire(0, &never_cancelled()) + // The whole point of repairing in place is that a failure must leave the old file + // where it was. Deleting first and writing after would open a window whose only + // surviving copy is the one the caller is about to destroy. + let err = store + .repair(&addr, b"not this chunk") .await - .expect("verify"); + .expect_err("must refuse"); + assert!(format!("{err}").contains("Refusing to repair"), "{err}"); assert!( - store.legacy_only_keys().is_empty(), - "a removed record must not come back on the copier's list" + path.exists(), + "the existing file must survive a refused repair" + ); + assert_eq!( + store.get(&addr).await.expect("get").expect("present"), + content ); + } + + #[tokio::test] + async fn a_chunk_can_be_deleted_and_stored_again() { + let (store, _dir) = test_store().await; + let (addr, content) = addressed("cycle"); + + assert!(store.put(&addr, &content).await.expect("put")); + assert!(store.delete(&addr).await.expect("delete")); assert!( - proof.is_clean(), - "and must not go on refusing the proof for ever" + store.put(&addr, &content).await.expect("re-put"), + "a re-stored chunk is new again" + ); + assert_eq!( + store.get(&addr).await.expect("get").expect("present"), + content ); } - #[test] - fn the_release_switches_are_never_written_to_an_operator_config_file() { - // A node writes its effective configuration back to disk. If these round-tripped, - // R1's values would be baked into every operator's file and the next release - // would change nothing. - let mut config = MigrationConfig::default(); - config.retire_legacy = !config.retire_legacy; - config.allow_shed = false; - config.shed_hold_hours = 5; - - let encoded = toml::to_string(&config).expect("encode"); - assert!(!encoded.contains("retire_legacy"), "{encoded}"); - - let decoded: MigrationConfig = toml::from_str(&encoded).expect("decode"); - let fresh = MigrationConfig::default(); - assert_eq!(decoded.retire_legacy, fresh.retire_legacy); - // Genuine operator controls do survive. - assert!(!decoded.allow_shed); - assert_eq!(decoded.shed_hold_hours, 5); + /// Write a chunk file straight into its shard, the way an existing store already + /// contains thousands of them. Bypasses the write path deliberately: this exercises + /// the startup scan, not `put`. + fn plant(chunks_dir: &Path, key: &XorName) { + let dir = chunks_dir.join(shard_name(key)); + std::fs::create_dir_all(&dir).expect("mkdir"); + std::fs::write(dir.join(hex::encode(key)), key).expect("plant"); } - #[test] - fn the_copy_order_is_closest_first() { - let me = [0u8; XORNAME_LEN_LOCAL]; - let mut near = [0u8; XORNAME_LEN_LOCAL]; - if let Some(b) = near.last_mut() { - *b = 1; + #[tokio::test] + async fn a_populated_and_churned_store_scans_correctly_at_scale() { + // Every shard populated, then aged the way a long-lived node ages: some keys + // deleted, others added in their place, so the directories carry holes rather + // than being freshly written. APFS enumeration is known to degrade with churn + // rather than with size, so a fresh corpus is not a realistic one. + const PLANTED: u32 = 20_000; + const CHURN: u32 = 1_000; + + let dir = TempDir::new().expect("temp dir"); + let chunks_dir = dir.path().join(CHUNKS_DIR_NAME); + std::fs::create_dir_all(&chunks_dir).expect("mkdir"); + + let mut expected: Vec = Vec::new(); + for i in 0..PLANTED { + let key = crate::client::compute_address(&i.to_le_bytes()); + plant(&chunks_dir, &key); + expected.push(key); } - let mut far = [0u8; XORNAME_LEN_LOCAL]; - if let Some(b) = far.first_mut() { - *b = 0xff; + for i in 0..CHURN { + let key = crate::client::compute_address(&i.to_le_bytes()); + std::fs::remove_file(chunks_dir.join(shard_name(&key)).join(hex::encode(key))) + .expect("churn out"); + let replacement = crate::client::compute_address(&(PLANTED + i).to_le_bytes()); + plant(&chunks_dir, &replacement); + } + expected.retain(|k| chunks_dir.join(shard_name(k)).join(hex::encode(k)).exists()); + for i in 0..CHURN { + expected.push(crate::client::compute_address(&(PLANTED + i).to_le_bytes())); } - let ordered = rank_closest_first(vec![far, near], Some(me)); - assert_eq!(ordered.first().copied(), Some(near)); - assert_eq!(ordered.last().copied(), Some(far)); - - // With no identity the order is still stable, which is all the copier needs. - let ordered = rank_closest_first(vec![far, near], None); - let mut expected = vec![far, near]; expected.sort_unstable(); - assert_eq!(ordered, expected); - } + expected.dedup(); - /// Local alias so the test does not import from the protocol crate. - const XORNAME_LEN_LOCAL: usize = 32; + let started = std::time::Instant::now(); + let store = reopen(&dir).await; + let scan = started.elapsed(); - #[test] - fn the_retirement_delay_can_never_be_shortened_below_the_retention_window() { - let config = MigrationConfig { - retire_delay_hours: 0, - ..MigrationConfig::default() - }; assert_eq!( - config.effective_retire_delay_hours(), - MIN_RETIRE_DELAY_HOURS + store.current_chunks().expect("count"), + expected.len() as u64 + ); + assert_eq!(store.all_keys().await.expect("all_keys"), expected); + + // Every shard should be in use at this size: 20,000 keys over 256 directories is + // about 78 each, and the last byte of a BLAKE3 output is uniform. + let occupied = std::fs::read_dir(&chunks_dir) + .expect("read store root") + .filter_map(std::result::Result::ok) + .filter(|e| e.file_name().to_str().is_some_and(|n| n.len() == 2)) + .count(); + assert_eq!(occupied, SHARD_COUNT, "the suffix must reach every shard"); + + println!( + "scan of {} keys across {SHARD_COUNT} shards took {scan:?}", + expected.len() ); } + + #[tokio::test] + async fn wait_idle_returns_once_writes_have_drained() { + let (store, _dir) = test_store().await; + let store = Arc::new(store); + for i in 0..32 { + let store = Arc::clone(&store); + let (addr, content) = addressed(&format!("drain-{i}")); + tokio::spawn(async move { store.put(&addr, &content).await }); + } + // Not a synchronisation point for tasks that have not been spawned yet, but it + // must not hang and it must leave the store usable. + store.wait_idle().await; + let (addr, content) = addressed("after-drain"); + assert!(store.put(&addr, &content).await.expect("put after drain")); + } } diff --git a/src/storage/file_store.rs b/src/storage/file_store.rs deleted file mode 100644 index 3172bdda..00000000 --- a/src/storage/file_store.rs +++ /dev/null @@ -1,3746 +0,0 @@ -//! One immutable file per chunk, content-addressed, with the filesystem as the -//! only authority. -//! -//! ```text -//! {root}/chunks/ store root -//! {root}/chunks/layout.json versioned layout marker -//! {root}/chunks/.lock advisory single-process guard -//! {root}/chunks//<64-hex> xy = the LAST two hex characters of the address -//! {root}/chunks//.tmp.. an in-flight write, in the destination directory -//! ``` -//! -//! # Why the *last* two hex characters -//! -//! A node holds keys for which it is among the [`CLOSE_GROUP_SIZE`] closest, so its -//! holdings share roughly `log2(N / CLOSE_GROUP_SIZE)` leading bits with its own node -//! ID, and that shared prefix grows as the network grows. Sharding on a prefix therefore -//! does not degrade, it collapses: at ~800 nodes a two-hex prefix already resolves to -//! about two distinct directories, and past a million nodes even a four-hex prefix -//! resolves to one. Close-group membership constrains the leading bits and places no -//! constraint at all on the trailing ones, and the address is a BLAKE3 output, so the -//! last byte is uniform by construction at every network size. -//! -//! 256 shards keeps a 24 GiB node at ~23 files per directory and a 1 TiB node at ~977, -//! for 1 MiB of directory inodes. The scheme and depth are recorded in `layout.json` at -//! creation so a future layout can be detected rather than silently misread. -//! -//! # Why lowercase hex names -//! -//! NTFS and default APFS fold case. Under an encoding with both cases (base64url, -//! base58) two distinct 32-byte keys can share one case-folded filename, which is a -//! silent overwrite. Hex has one case-folded form per key, and no hex string can ever -//! spell a reserved Windows device name (`CON`, `NUL`, `AUX`, `COM1`, ...) because none -//! of those letters is in `0-9a-f`. The full 64-character key stays in the filename, so -//! a `find` over the tree recovers the whole store even if the directory layer is lost. -//! -//! [`CLOSE_GROUP_SIZE`]: crate::ant_protocol::CLOSE_GROUP_SIZE - -use crate::ant_protocol::{XorName, MAX_CHUNK_SIZE, XORNAME_LEN}; -use crate::error::{Error, Result}; -use crate::logging::{debug, info, trace, warn}; -use crate::storage::StorageStats; -use fs2::FileExt; -use serde::{Deserialize, Serialize}; -use std::collections::{BTreeSet, HashMap, HashSet}; -use std::fs::{File, OpenOptions}; -use std::io::{ErrorKind, Read, Write}; -use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::Arc; -use std::time::{Duration, Instant}; -use tokio::task::spawn_blocking; -use tokio_util::task::TaskTracker; - -/// Directory under the node root that holds the chunk files. -pub const CHUNKS_DIR_NAME: &str = "chunks"; - -/// Name of the layout marker written once at store creation. -pub const LAYOUT_FILE_NAME: &str = "layout.json"; - -/// Name of the advisory single-process lock file. -const LOCK_FILE_NAME: &str = ".lock"; - -/// Prefix that marks an in-flight write. Never a valid chunk name (chunk names are -/// exactly [`CHUNK_NAME_LEN`] lowercase hex characters, and `.` is not hex). -const TEMP_PREFIX: &str = ".tmp."; - -/// Number of shard directories. One level, `00` through `ff`. -const SHARD_COUNT: usize = 256; - -/// Length of a chunk filename: the full address in lowercase hex. -const CHUNK_NAME_LEN: usize = XORNAME_LEN * 2; - -/// How often to re-query available disk space, in seconds. -/// -/// Matches the LMDB store's cadence so the capacity predicate behaves identically -/// for callers that only ask "is there room at all". -const DISK_CHECK_INTERVAL_SECS: u64 = 5; - -/// Allocation granularity assumed when charging a pending write against free space. -/// -/// Every filesystem we support allocates in units of at least 4 KiB, so a write of -/// `n` bytes consumes at least `ceil(n / 4096) * 4096`. One extra unit covers the -/// directory entry and inode. -const ALLOC_UNIT: u64 = 4096; - -/// How many times a publish retries a transient Windows sharing violation. -const RENAME_RETRY_ATTEMPTS: u32 = 5; - -/// Base backoff between those retries; the wait grows linearly with the attempt. -const RENAME_RETRY_BACKOFF: Duration = Duration::from_millis(20); - -/// Longest absolute path a chunk file may need, checked once at open. -/// -/// Windows caps a non-verbatim path at `MAX_PATH` (260) including the terminating NUL. -/// Rust's standard library transparently switches to the `\\?\` verbatim form for long -/// absolute paths, so this is a warning rather than a hard failure, but an operator who -/// buries the node root ten directories deep should hear about it before the first write -/// fails rather than after. -#[cfg(windows)] -const WINDOWS_PATH_WARN_LEN: usize = 240; - -/// The on-disk layout marker. -/// -/// Written once when the store directory is created and read on every subsequent open. -/// Nothing in this survey of comparable stores (IPFS flatfs, Storj, borgbackup) shipped -/// an in-place re-sharder, and all three paid for it. Recording the scheme costs one -/// small file and is the difference between changing the default later and never being -/// able to. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct StoreLayout { - /// Marker schema version. A store written by a newer schema is refused. - pub schema: u32, - /// How a chunk address maps to a shard directory. - pub scheme: String, - /// How many hex characters of the address name the shard directory. - pub shard_chars: u8, - /// How many directory levels of sharding. - pub depth: u8, - /// How a chunk address maps to a filename. - pub name_encoding: String, -} - -/// Marker schema this build writes and understands. -const LAYOUT_SCHEMA: u32 = 1; -/// Shard scheme this build implements: the trailing hex characters of the address. -const LAYOUT_SCHEME_SUFFIX_HEX: &str = "suffix-hex"; -/// Filename encoding this build implements. -const LAYOUT_NAME_LOWER_HEX: &str = "lower-hex"; - -impl Default for StoreLayout { - fn default() -> Self { - Self { - schema: LAYOUT_SCHEMA, - scheme: LAYOUT_SCHEME_SUFFIX_HEX.to_string(), - shard_chars: 2, - depth: 1, - name_encoding: LAYOUT_NAME_LOWER_HEX.to_string(), - } - } -} - -impl StoreLayout { - /// Return an error unless this build can read a store written with this layout. - fn check_supported(&self) -> Result<()> { - if self.schema > LAYOUT_SCHEMA { - return Err(Error::Storage(format!( - "Chunk store layout schema {} is newer than this build understands ({LAYOUT_SCHEMA}). \ - Refusing to open rather than misread the store.", - self.schema - ))); - } - if self.scheme != LAYOUT_SCHEME_SUFFIX_HEX { - return Err(Error::Storage(format!( - "Chunk store uses shard scheme '{}', this build implements '{LAYOUT_SCHEME_SUFFIX_HEX}'", - self.scheme - ))); - } - if self.shard_chars != 2 || self.depth != 1 { - return Err(Error::Storage(format!( - "Chunk store uses {} shard characters at depth {}, this build implements 2 at depth 1", - self.shard_chars, self.depth - ))); - } - if self.name_encoding != LAYOUT_NAME_LOWER_HEX { - return Err(Error::Storage(format!( - "Chunk store names files with '{}', this build implements '{LAYOUT_NAME_LOWER_HEX}'", - self.name_encoding - ))); - } - Ok(()) - } -} - -/// Configuration for [`FileStore`]. -#[derive(Debug, Clone)] -pub struct FileStoreConfig { - /// Node root directory. The store lives at `{root_dir}/chunks/`. - pub root_dir: PathBuf, - /// Verify `BLAKE3(content) == address` on read. - pub verify_on_read: bool, - /// Free bytes to keep on the storage partition. Writes are refused below this. - pub disk_reserve: u64, -} - -/// Outcome of a single write attempt, used to keep the duplicate accounting honest. -enum PutOutcome { - /// The chunk was newly published. - New, - /// The chunk was already on disk. - Duplicate, -} - -/// Snapshot of free space, plus what has been written since it was taken. -#[derive(Debug)] -struct CapacitySnapshot { - /// When `available` was measured. `None` means never. - measured_at: Option, - /// Free bytes reported by the filesystem at `measured_at`. - available: u64, - /// Bytes published since `measured_at`, charged against `available`. - /// - /// Cleared by a fresh measurement, which already accounts for them. - written_since: u64, - /// Bytes reserved by writes that have not landed yet. - /// - /// Deliberately **not** cleared by a measurement: a `statvfs` taken while writes are - /// in flight reports space those writes are about to consume, so forgetting their - /// reservations at that moment would hand the same bytes out twice. That is precisely - /// the over-admission the reservation exists to prevent. - in_flight: u64, -} - -/// Size-aware free-space predicate with a short-lived cache. -/// -/// Free bytes alone stopped being a sufficient answer the moment chunks became files: -/// a caller wants to know whether *this* write fits, not whether the disk is non-empty. -/// The cache keeps the common case at one `statvfs` per interval while staying correct -/// under a burst, because bytes written since the measurement are charged against it. -#[derive(Debug)] -struct CapacityGuard { - /// Directory whose partition is measured. - dir: PathBuf, - /// Free bytes to keep unused. - reserve: u64, - /// The cached measurement. - snapshot: parking_lot::Mutex, -} - -impl CapacitySnapshot { - /// Free bytes, less everything written or promised since the measurement. - fn free_estimate(&self) -> u64 { - self.available - .saturating_sub(self.written_since) - .saturating_sub(self.in_flight) - } -} - -impl CapacityGuard { - /// Create a guard over the partition hosting `dir`. - fn new(dir: PathBuf, reserve: u64) -> Self { - Self { - dir, - reserve, - snapshot: parking_lot::Mutex::new(CapacitySnapshot { - measured_at: None, - available: 0, - written_since: 0, - in_flight: 0, - }), - } - } - - /// Bytes actually consumed on disk by a payload of `len` bytes. - fn charge(len: u64) -> u64 { - // Round the payload up to the allocation unit, then add one unit for the - // directory entry and inode. - len.div_ceil(ALLOC_UNIT) - .saturating_mul(ALLOC_UNIT) - .saturating_add(ALLOC_UNIT) - } - - /// Free bytes right now, or `None` if the question could not be answered. - /// - /// Deliberately separate from [`Self::measure`], which folds a failure into an error - /// the caller cannot tell from "below the reserve". - fn measure_available(&self) -> Option { - let mut snapshot = self.snapshot.lock(); - match self.measure(&mut snapshot) { - Ok(()) => Some(snapshot.free_estimate()), - Err(_) => None, - } - } - - /// Query the filesystem and refresh the snapshot. - fn measure(&self, snapshot: &mut CapacitySnapshot) -> Result<()> { - let available = fs2::available_space(&self.dir) - .map_err(|e| Error::Storage(format!("Failed to query available disk space: {e}")))?; - snapshot.available = available; - // Reservations survive: their bytes are not on the platter yet, so the fresh - // measurement does not include them. - snapshot.written_since = 0; - snapshot.measured_at = Some(Instant::now()); - Ok(()) - } - - /// Test `needed` against the snapshot, refreshing it if it is stale or short. - /// - /// Only *passing* results are cached, so a low-space condition is rechecked on every - /// call and freed space is noticed promptly. - fn admit(&self, snapshot: &mut CapacitySnapshot, needed: u64) -> Result<()> { - let want = self.reserve.saturating_add(Self::charge(needed)); - - let cache_fresh = snapshot - .measured_at - .is_some_and(|t| t.elapsed().as_secs() < DISK_CHECK_INTERVAL_SECS); - if cache_fresh && snapshot.free_estimate() >= want { - return Ok(()); - } - - self.measure(snapshot)?; - if snapshot.free_estimate() < want { - // Do not cache a failing result: `measured_at` is left set so the next call - // still re-measures, because the branch above only short-circuits a pass. - return Err(Error::Storage(format!( - "Insufficient disk space: {:.2} GiB available, {:.2} GiB reserve required. \ - Free disk space or increase the partition to continue storing chunks.", - bytes_to_gib(snapshot.free_estimate()), - bytes_to_gib(self.reserve), - ))); - } - Ok(()) - } - - /// Drop the cached measurement so the next question hits the filesystem. - fn invalidate(&self) { - let mut snapshot = self.snapshot.lock(); - snapshot.measured_at = None; - snapshot.written_since = 0; - } - - /// Return `Ok(())` if a write of `needed` bytes would fit. Charges nothing. - fn check(&self, needed: u64) -> Result<()> { - let mut snapshot = self.snapshot.lock(); - self.admit(&mut snapshot, needed) - } - - /// Admit a write of `needed` bytes and charge it in the same critical section. - /// - /// Checking and charging separately is the bug this exists to prevent: dozens of - /// protocol handlers can each pass against the same cached measurement before any of - /// them has written a byte, and collectively cross the reserve. - /// - /// The returned [`Reservation`] settles itself when dropped, so a caller whose future - /// is dropped mid-write cannot strand it. Nothing else ever decrements the in-flight - /// count, so a stranded reservation would be permanent, and enough of them would make - /// an empty disk look full until the process restarted. - fn reserve(self: &Arc, needed: u64) -> Result { - { - let mut snapshot = self.snapshot.lock(); - self.admit(&mut snapshot, needed)?; - snapshot.in_flight = snapshot.in_flight.saturating_add(Self::charge(needed)); - } - Ok(Reservation { - capacity: Arc::clone(self), - bytes: needed, - settled: false, - }) - } - - /// The two accounting counters, for tests that need to prove a charge was - /// settled one way rather than the other. - #[cfg(test)] - fn counters(&self) -> (u64, u64) { - let snapshot = self.snapshot.lock(); - (snapshot.written_since, snapshot.in_flight) - } - - /// Give back a reservation whose write did not happen. - fn release(&self, needed: u64) { - let mut snapshot = self.snapshot.lock(); - snapshot.in_flight = snapshot.in_flight.saturating_sub(Self::charge(needed)); - } - - /// Turn a reservation into bytes that are now on disk. - fn commit_reservation(&self, needed: u64) { - let charge = Self::charge(needed); - let mut snapshot = self.snapshot.lock(); - snapshot.in_flight = snapshot.in_flight.saturating_sub(charge); - snapshot.written_since = snapshot.written_since.saturating_add(charge); - } - - /// Credit a completed delete back to the cached measurement. - fn record_removed(&self, len: u64) { - let mut snapshot = self.snapshot.lock(); - snapshot.written_since = snapshot.written_since.saturating_sub(Self::charge(len)); - } -} - -/// A charged, unsettled write. -/// -/// Held by whatever is actually doing the write, so the charge is released even if the -/// caller's future is dropped and only the blocking closure survives. -pub(crate) struct Reservation { - /// The guard this was taken from. - capacity: Arc, - /// Payload size, before rounding. - bytes: u64, - /// Whether it has already been accounted for. - settled: bool, -} - -impl Reservation { - /// The write landed: move the charge from in-flight to written. - pub(crate) fn commit(mut self) { - self.capacity.commit_reservation(self.bytes); - self.settled = true; - } -} - -impl Drop for Reservation { - fn drop(&mut self) { - if !self.settled { - self.capacity.release(self.bytes); - } - } -} - -/// Environment variable naming a failpoint: stop after the temp file, before the rename. -#[cfg(any(test, feature = "test-utils"))] -pub const HALT_BEFORE_PUBLISH: &str = "ANT_HALT_BEFORE_PUBLISH"; - -/// Environment variable naming a failpoint: stop once the legacy environment is renamed -/// aside and marked retired, before any of it is deleted. -/// -/// The most destructive window in the migration. What a start that finds a marked -/// directory must do is finish the deletion, never reopen it, because the node has already -/// told the network it holds those chunks from the file store. -#[cfg(any(test, feature = "test-utils"))] -pub const HALT_AFTER_RETIRE_MARK: &str = "ANT_HALT_AFTER_RETIRE_MARK"; - -/// Park forever at a named failpoint, once a marker says the process has reached it. -/// -/// For crash tests, which need a process to die *inside* an operation rather than at -/// whatever point a sleep in another process happened to land. The variable holds a path: -/// this writes it, so the parent knows the child is exactly here, and then waits to be -/// killed. -/// -/// Costs one environment read per write when the feature is compiled in, and the feature -/// is not in a release build. -#[cfg(any(test, feature = "test-utils"))] -pub(crate) fn halt_here_if_asked(variable: &str, reached: &Path) { - let Ok(marker) = std::env::var(variable) else { - return; - }; - // Let the first few through. A test that stops the very first write leaves a store - // with nothing successfully in it, and an assertion over what it holds then passes by - // iterating nothing. Letting some land first means the crash happens to a store that - // has real chunks in it, which is the situation worth checking. - let skip: u64 = std::env::var(HALT_AFTER) - .ok() - .and_then(|raw| raw.parse().ok()) - .unwrap_or(0); - if HALTS_SEEN.fetch_add(1, std::sync::atomic::Ordering::AcqRel) < skip { - return; - } - if let Err(e) = std::fs::write(&marker, reached.as_os_str().as_encoded_bytes()) { - // The parent waits for this file. Saying so on the way past is the difference - // between a test that fails and one that hangs until the job times out. - eprintln!("failpoint could not write its marker {marker}: {e}"); - return; - } - loop { - std::thread::sleep(Duration::from_hours(1)); - } -} - -/// How many writes to let through before the failpoint fires. -#[cfg(any(test, feature = "test-utils"))] -pub const HALT_AFTER: &str = "ANT_HALT_AFTER"; - -/// How many times the failpoint has been reached in this process. -#[cfg(any(test, feature = "test-utils"))] -static HALTS_SEEN: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); - -/// Clears a write's registration when the work finishes, however it finishes. -/// -/// Held by the blocking closure rather than by the caller, so a dropped future cannot -/// leave an entry behind, and a panic in the work cannot either. -struct WriteInFlight { - writing: Arc>>, - finished: Arc, - address: XorName, -} - -impl Drop for WriteInFlight { - fn drop(&mut self) { - let was_last = { - let mut writing = self.writing.lock(); - match writing.get_mut(&self.address) { - Some(count) if *count > 1 => { - *count -= 1; - false - } - _ => { - writing.remove(&self.address); - true - } - } - }; - // Only when this was the last one. Waking a waiter while another write for the - // same key is still queued is exactly what the count exists to prevent. - if was_last { - self.finished.notify_waiters(); - } - } -} - -/// What is behind a chunk's name on disk. -/// -/// Four answers, not two, because "could not read it" must never be treated as "wrong": -/// replacing a chunk is destructive, and off Unix it truncates the file in place, so a -/// transient fault would turn a healthy sole copy into an empty one. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum StoredBytes { - /// The bytes are there and hash to the name. - Good, - /// The bytes are there and do not. - Wrong, - /// There is nothing behind the name. - Absent, - /// The question could not be answered this time. - Unreadable, -} - -/// Convert a byte count to GiB for human-readable log messages. -#[allow(clippy::cast_precision_loss)] // display only — sub-byte precision is irrelevant -fn bytes_to_gib(bytes: u64) -> f64 { - bytes as f64 / (1024.0 * 1024.0 * 1024.0) -} - -/// Content-addressed store holding one immutable file per chunk. -/// -/// The filesystem is the sole authority. The in-memory index is a cache of what the -/// directory tree already contains, rebuilt from directory entries at every open, and -/// every mutation of it mirrors a filesystem operation that has *already* completed. -/// Bitcask's issue #114 is the cautionary tale for the opposite order: an index that is -/// rebuilt at startup and then mutated in anticipation drifts, and the drift is silent. -#[derive(Debug)] -pub struct FileStore { - /// Store configuration. - config: FileStoreConfig, - /// `{root_dir}/chunks`. - chunks_dir: PathBuf, - /// Every address whose file is published, in ascending order. - /// - /// `BTreeSet` rather than a hash set because `all_keys()` must be sorted (the - /// commitment builder truncates with `take(cap)` *before* the Merkle tree sorts, so - /// an unstable order would make the node's published commitment depend on iteration - /// luck), and because it never spikes memory while growing. - index: Arc>>, - /// One mutex per shard, serialising writers of the same address. - /// - /// LMDB gave exactly-once `put` semantics for free: the duplicate test happened - /// inside the write transaction. Two threads publishing the same address here would - /// otherwise both see an absent file, both rename, and both report "newly stored", - /// double-counting the chunk. The lane is indexed by the address's LAST byte for the - /// same reason the shard is: a node's keys share their leading bytes, so lanes keyed - /// on the first byte would all collapse into one. - write_lanes: Arc>>, - /// Operation counters, same shape as the LMDB store reported. - stats: parking_lot::RwLock, - /// Which of the 256 shard directories are known to exist, so a steady-state write - /// does not pay a `create_dir_all` syscall. - shards_present: Arc>, - /// Indexed chunks this store currently cannot read. - /// - /// Held back from everything the node says it has, while the files themselves are - /// left alone. See [`Self::mark_suspect`]. - suspect: Arc>>, - /// Indexed chunks a read has proven do not match their name. - /// - /// Separate from the above because they clear differently. Not being able to read a - /// file is a question a later read answers; bytes that are wrong stay wrong however - /// often they are read, and only a repair or a removal settles it. A raw read that - /// does not hash anything must not take a chunk out of this set. - known_wrong: Arc>>, - /// Addresses this store is part-way through writing. - /// - /// Every mutation registers here before it spawns its blocking work and clears the - /// entry *inside* that work, so a caller whose future is dropped cannot skip the - /// clearing while the write itself goes on to land. That is the difference that - /// matters: the blocking half is not cancelled with the future, so anything the - /// future was going to do afterwards is not a record of what happened. - /// - /// It lets a delete queue behind the exact write it would otherwise race, rather than - /// behind every write this store has in flight. - /// - /// Counted, not a set. Cancellation can release the facade's key lane while the - /// blocking half survives, so a second write for the same key can start behind the - /// first. With one entry between them, whichever finished first would remove it and a - /// waiter would be told the key is free while the other was still queued. - writing: Arc>>, - /// Woken when [`Self::writing`] loses its last entry for a key. - write_finished: Arc, - /// Bumped whenever a chunk stops being servable. - /// - /// The pre-retirement pass reads every chunk, and its result is reused for a while - /// rather than re-read on every tick. This is how the caller can tell that the store - /// has not changed underneath that result: a proof carries the value it saw, and a - /// file that has since gone or stopped being readable makes it stale. - health: Arc, - /// Size-aware free-space predicate. - capacity: Arc, - /// Monotonic counter that makes temp filenames unique within this store. - temp_seq: AtomicU64, - /// Random per-instance discriminator for temp filenames. - nonce: u32, - /// Held for the store's lifetime. Startup fails without it. - /// - /// Shared rather than owned so the blocking work that depends on it can hold a lease - /// of its own: that work outlives the future that spawned it, and a cancelled caller - /// releasing the lock would leave it writing into a directory another process had - /// just been let into. - lock: Arc, - /// Tracks every blocking task, so [`FileStore::wait_idle`] can wait for writes that - /// outlived their awaiting future. - blocking_tracker: TaskTracker, - /// Test-only gate read-acquired at the top of the put blocking closure. - /// - /// Tests hold the write half to park an in-flight write on the blocking pool, which - /// is the shape a `select!` losing to a shutdown token leaves behind. - #[cfg(any(test, feature = "test-utils"))] - test_put_gate: Arc>, -} - -impl FileStore { - /// Open (or create) the store at `{root_dir}/chunks/`. - /// - /// Sweeps orphaned temp files, then rebuilds the index from directory entries. - /// The scan reads names only: it never `stat`s an entry and never reads a chunk. - /// - /// # Errors - /// - /// Returns [`Error::Storage`] if the directory cannot be created, the layout marker - /// is unreadable or describes a layout this build does not implement, or the scan - /// fails. - pub async fn new(config: FileStoreConfig) -> Result { - let chunks_dir = config.root_dir.join(CHUNKS_DIR_NAME); - std::fs::create_dir_all(&chunks_dir).map_err(|e| { - Error::Storage(format!( - "Failed to create chunk store directory {}: {e}", - chunks_dir.display() - )) - })?; - - check_path_budget(&chunks_dir); - - let layout = read_or_write_layout(&chunks_dir)?; - layout.check_supported()?; - - // Startup fails without it, so from here this process is the only one using this - // directory and an interrupted write can only be its own. - let lock = acquire_store_lock(&chunks_dir)?; - - let scan_dir = chunks_dir.clone(); - // The scan holds the lease itself. It sweeps interrupted writes on the strength of - // being alone here, and it runs on a thread that outlives this future: a - // cancelled startup that released the lock would leave it sweeping a directory - // another process had just been let into. - let scan_lease = Arc::clone(&lock); - // The node root as well as the chunk tree. The scan sweeps interrupted writes - // under `chunks/`, which covers the layout marker's temporary because that lives - // there; the migration marker's lives in the root, where nothing looked. - let root = config.root_dir.clone(); - let scan = spawn_blocking(move || { - let _lease = scan_lease; - sweep_marker_temps(&root); - scan_store(&scan_dir) - }) - .await - .map_err(|e| Error::Storage(format!("Chunk store scan task failed: {e}")))??; - - let ScanResult { - keys, - shards_present, - swept_temps, - skipped, - } = scan; - - let key_count = keys.len(); - // Build from a sorted vector: bulk-building packs every B-tree node to its - // capacity, where repeated `insert` converges on ~68% fill for the same keys. - let index: BTreeSet = keys.into_iter().collect(); - - if swept_temps > 0 { - info!("Chunk store: removed {swept_temps} orphaned temporary file(s) from interrupted writes"); - } - if skipped > 0 { - warn!("Chunk store: ignored {skipped} directory entr(ies) that are not chunk files"); - } - info!( - "Chunk store open at {} ({key_count} chunks)", - chunks_dir.display() - ); - - let capacity = Arc::new(CapacityGuard::new(chunks_dir.clone(), config.disk_reserve)); - - Ok(Self { - config, - chunks_dir, - index: Arc::new(parking_lot::RwLock::new(index)), - write_lanes: Arc::new( - std::iter::repeat_with(|| parking_lot::Mutex::new(())) - .take(SHARD_COUNT) - .collect(), - ), - stats: parking_lot::RwLock::new(StorageStats::default()), - shards_present: Arc::new(parking_lot::Mutex::new(shards_present)), - suspect: Arc::new(parking_lot::RwLock::new(HashSet::new())), - known_wrong: Arc::new(parking_lot::RwLock::new(HashSet::new())), - writing: Arc::new(parking_lot::Mutex::new(HashMap::new())), - write_finished: Arc::new(tokio::sync::Notify::new()), - health: Arc::new(std::sync::atomic::AtomicU64::new(0)), - capacity, - temp_seq: AtomicU64::new(0), - nonce: rand::random(), - lock, - blocking_tracker: TaskTracker::new(), - #[cfg(any(test, feature = "test-utils"))] - test_put_gate: Arc::new(parking_lot::RwLock::new(())), - }) - } - - /// Store a chunk. - /// - /// On Unix, publishing is a rename within the destination directory, so the final name - /// can never appear on partial content: the name *is* the hash, and the content is - /// fully written and flushed before the name exists. Off Unix there is no rename, for - /// the reason `publish_in_place` gives (it is compiled only on those platforms, so this - /// is not a link), and a partial file can wear a real name; that - /// is why a duplicate is read and compared rather than trusted. - /// - /// # Returns - /// - /// `true` if the chunk was newly stored, `false` if it was already present. - /// - /// # Errors - /// - /// Returns [`Error::Storage`] if the content does not hash to `address`, the disk - /// is too full, or the write fails. - pub async fn put(&self, address: &XorName, content: &[u8]) -> Result { - let computed = crate::client::compute_address(content); - if computed != *address { - return Err(Error::Storage(format!( - "Content address mismatch: expected {}, computed {}", - hex::encode(address), - hex::encode(computed) - ))); - } - // The read path refuses anything over the ceiling, so writing one would create a - // file the store could never read back and could never repair. - if content.len() > MAX_CHUNK_SIZE { - return Err(Error::Storage(format!( - "Chunk {} is {} bytes, over the {MAX_CHUNK_SIZE} byte maximum", - hex::encode(address), - content.len() - ))); - } - - // An indexed name is not proof of the bytes under it. The index is built from - // names, by the startup scan and by a completed publish, and a name can outlive - // what it points at: off Unix a chunk is created under its final name before its - // bytes are written, so a crash leaves a short file wearing a real name, and rot - // leaves a full-length one. Answering "already have it" to the copy that would fix - // either is how a node discards its own repair and is never offered another. - // - // So the bytes decide. Checked before the reservation below, so re-storing a chunk - // this node already holds stays a no-op on a full disk. - if self.index.read().contains(address) { - if let Some(answer) = self.settle_indexed_duplicate(address, content).await { - return answer; - } - } - - let len = content.len() as u64; - // Reserved after the duplicate test so re-storing an existing chunk stays a - // harmless no-op on a full disk, matching the LMDB store's ordering. - let reservation = self.capacity.reserve(len)?; - - let shard = self.chunks_dir.join(shard_name(address)); - let final_path = shard.join(hex::encode(address)); - let temp_path = shard.join(self.next_temp_name()); - let payload = content.to_vec(); - let lanes = Arc::clone(&self.write_lanes); - let index = Arc::clone(&self.index); - let shards_present = Arc::clone(&self.shards_present); - let chunks_dir = self.chunks_dir.clone(); - let lane = shard_index(address); - let key = *address; - #[cfg(any(test, feature = "test-utils"))] - let test_put_gate = Arc::clone(&self.test_put_gate); - // Registered before the work is spawned and cleared by the work itself, so a - // caller that goes away cannot leave a delete free to race this publish. - let in_flight = self.begin_write(address); - // And the lease, for the same reason the scan holds it: this thread writes into a - // directory whose exclusivity the lock is what establishes, and it can outlive - // the last owner of the store. - let lease = Arc::clone(&self.lock); - let known_wrong = Arc::clone(&self.known_wrong); - let suspect = Arc::clone(&self.suspect); - - let outcome = self - .blocking_tracker - .spawn_blocking(move || -> Result { - let _in_flight = in_flight; - let _lease = lease; - // Test-only: parks here while a test holds the write half. - #[cfg(any(test, feature = "test-utils"))] - let _test_put_gate = test_put_gate.read(); - let _lane = lanes.get(lane).map(parking_lot::Mutex::lock); - // `mkdir` plus a directory flush are syscalls, so they belong here and - // not on a runtime worker. - ensure_shard_dir(&chunks_dir, &shard, lane, &shards_present)?; - let outcome = match publish(&temp_path, &final_path, &payload, &shard) { - Ok(outcome) => outcome, - Err(PublishFailed { error, left_behind }) => { - // A publish that failed can still have left the bytes there: off - // Unix the chunk is created under its final name, and if the write - // or the flush then fails, the cleanup that removes it can fail - // too. Releasing the reservation would hand back a charge for a - // file that is on the disk. - // - // The publish says so rather than this deciding from a later - // `is_file`. Asking the filesystem afterwards infers ownership from - // a name being occupied, which is true under the store lock and the - // shard lane and not true against anything out of band, and this - // file spends a lot of its length arguing that a name is not - // evidence. A bit set by the code that created the file is. - if left_behind { - reservation.commit(); - } - return Err(error); - } - }; - // Placed, not yet durable. A failure from here on leaves the bytes on the - // disk: the chunk is rightly not reported as stored, because a copy that is - // not durable must not authorise deleting another, but the space is spent - // all the same. Dropping the reservation would hand that charge back and - // admit the next write against room that is already gone. - // - // Only for a chunk this call published. `Duplicate` means the file was - // already there and was charged by whoever wrote it, so charging it again - // here would count one file twice and shrink the store's idea of its own - // disk on every retry. - if let Err(e) = flush_publication(&final_path, &shard) { - if matches!(outcome, PutOutcome::New) { - reservation.commit(); - } - return Err(e); - } - // Index inside the lane, and only after the rename has returned. A - // concurrent delete of the same address therefore cannot interleave - // between publishing the file and admitting the key. - // - // Only for a chunk this call actually published. `Duplicate` says a file - // already wears the name, and a name is not evidence about the bytes under - // it: the four-way answer that decides whether they are good, wrong, absent - // or unreadable runs after the await below, and a caller whose future is - // dropped never reaches it. Admitting the key here would leave the node - // claiming, advertising and committing to bytes nothing has read, with no - // suspect or known-wrong mark to hold it back, and the sharpest case is a - // name the startup scan deliberately refused because what wears it is a - // fifo, a socket or a directory. The duplicate arm admits the key itself, - // once a read has proven the bytes. - if matches!(outcome, PutOutcome::New) { - index.write().insert(key); - // With the marks that would otherwise hold the key back. These bytes - // were hashed against their own name on the way in, so an older - // instance proven wrong or merely unreadable has just been replaced by - // a good one. Cleared here rather than after the await for the same - // reason the insert is here: a cancelled caller would leave the key - // indexed and suppressed at once, so a chunk this node really does hold - // would stay hidden from `exists` and `all_keys` until some later read - // happened to settle it. - known_wrong.write().remove(&key); - suspect.write().remove(&key); - // Settled here, inside the work, so a dropped awaiter cannot strand it. - reservation.commit(); - } - Ok(outcome) - }) - .await - .map_err(|e| Error::Storage(format!("Chunk store put task failed: {e}")))??; - - match outcome { - PutOutcome::Duplicate => self.settle_duplicate(address, content).await, - PutOutcome::New => { - // Freshly published bytes that were checked against their own name on the - // way in. The marks were already cleared inside the work, where a dropped - // caller cannot skip them; what is left here is only what a caller who is - // still waiting should see. - let mut stats = self.stats.write(); - stats.chunks_stored = stats.chunks_stored.saturating_add(1); - stats.bytes_stored = stats.bytes_stored.saturating_add(len); - drop(stats); - debug!("Stored chunk {} ({len} bytes)", hex::encode(address)); - Ok(true) - } - } - } - - /// Decide what a name that was already taken actually means. - /// - /// Split out of [`Self::put`] because it is a different question. `put` puts bytes on - /// a disk; this reads bytes back to find out whether the ones already there are the - /// ones the caller is offering, which is the only thing that makes a duplicate safe to - /// report as stored. - /// - /// # Errors - /// - /// Returns [`Error::Storage`] when the existing file is absent or could not be read, - /// both of which mean this node must not report the chunk as held. - async fn settle_duplicate(&self, address: &XorName, content: &[u8]) -> Result { - // The file was already on disk, and its name is not evidence its contents - // are right. The startup scan indexes by name without reading anything, - // and on Windows a crash mid-write leaves a partial file under a real - // chunk name. Trusting the name here would acknowledge a chunk that was - // never stored, and then discard the good copy arriving to repair it. - // Every answer handled, because three of the four must not report the - // chunk as stored. A caller that hears success acts on it: a client drops - // its own copy, replication marks the key held, and the copier takes it - // out of the legacy-only set. - match self.stored_bytes_match(address).await { - StoredBytes::Good => { - // Admitted here, which is the first moment the bytes behind the - // name have been read and shown to hash to it. Idempotent: the - // ordinary case is a key the startup scan already indexed. - self.index.write().insert(*address); - { - let mut stats = self.stats.write(); - stats.duplicates = stats.duplicates.saturating_add(1); - } - Ok(false) - } - StoredBytes::Wrong => { - warn!( - "Chunk {} was already on disk but its contents are wrong; \ - replacing it with the copy just offered", - hex::encode(address) - ); - self.repair(address, content).await.map(|()| true) - } - // The name was taken a moment ago and is not now, or was never a - // readable chunk file. Either way nothing holds these bytes, so say so - // rather than reporting a chunk that is not there. - StoredBytes::Absent => Err(Error::Storage(format!( - "Chunk {} was reported already on disk but nothing is there. Not \ - reporting it as stored.", - hex::encode(address) - ))), - // Replacing on an unanswered question would destroy a healthy copy, - // and reporting success would discard the offered one. The index entry - // stays: the file is still there, and dropping the entry would leave - // the chunk in neither this store's view nor the legacy one, which is - // what retirement destroys. Removing an entry is the quarantine path's - // job, and it removes the file with it, after a read that succeeded - // and proved the bytes wrong. - StoredBytes::Unreadable => Err(Error::Storage(format!( - "Chunk {} is on disk but could not be read to check it. Not \ - replacing it, and not reporting it as stored.", - hex::encode(address) - ))), - } - } - - /// Flush every directory a chunk can live in, so the names in them are durable. - /// - /// Byte integrity is not the whole of what the pre-retirement proof has to establish. - /// A chunk whose contents are on the platter but whose *name* is not is still lost to - /// a power loss, and a publish whose rename landed and whose directory flush failed - /// leaves exactly that: the next attempt sees the name, the next verification reads - /// the right bytes, and nothing goes back to retry the flush. So the proof flushes - /// them itself rather than trusting that each publish did. - /// - /// Cheap: at most 257 directory flushes for a store of any size, and nothing off Unix, - /// where directories cannot be flushed and the retirement marker covers the same - /// ground instead. - /// - /// # Errors - /// - /// Returns [`Error::Storage`] on the first directory that cannot be flushed. The - /// caller must treat that as a proof it did not get. - pub fn flush_namespace(&self) -> Result<()> { - fsync_dir(&self.chunks_dir).map_err(|e| { - Error::Storage(format!( - "Could not flush {}: {e}", - self.chunks_dir.display() - )) - })?; - let present = *self.shards_present.lock(); - for (shard, _) in present.iter().enumerate().filter(|(_, here)| **here) { - let dir = self.chunks_dir.join(format!("{shard:02x}")); - fsync_dir(&dir) - .map_err(|e| Error::Storage(format!("Could not flush {}: {e}", dir.display())))?; - } - Ok(()) - } - - /// Decide what to do about a write of a chunk the index already names. - /// - /// `None` means the index was wrong and there is nothing on disk, so the caller - /// publishes it as new. Everything else is the answer. - async fn settle_indexed_duplicate( - &self, - address: &XorName, - content: &[u8], - ) -> Option> { - match self.stored_bytes_match(address).await { - StoredBytes::Good => { - trace!("Chunk {} already exists", hex::encode(address)); - { - let mut stats = self.stats.write(); - stats.duplicates = stats.duplicates.saturating_add(1); - } - Some(Ok(false)) - } - StoredBytes::Wrong => { - warn!( - "Chunk {} is indexed but its bytes are wrong; replacing it with the \ - copy just offered", - hex::encode(address) - ); - Some(self.repair(address, content).await.map(|()| true)) - } - // Indexed but gone: publish it fresh rather than replacing something that is - // not there. - StoredBytes::Absent => None, - // Unanswerable this time. Do not touch what is there, and do not tell the - // caller the chunk is safely stored either: a client would take that as an - // acknowledgement and drop the only other copy. The index entry stays, for - // the reason given on the same case after publication. - StoredBytes::Unreadable => Some(Err(Error::Storage(format!( - "Chunk {} is indexed but could not be read to check it. Not replacing it, \ - and not reporting it as stored.", - hex::encode(address) - )))), - } - } - - /// Drop an address from the index without touching the file. Tests only. - /// - /// Stands in for whatever leaves a key indexed nowhere: a quarantine, a publish that - /// failed after the file went, an operator with a shell. - #[cfg(test)] - pub(crate) fn forget_for_test(&self, address: &XorName) { - self.index.write().remove(address); - } - - /// The size of the file behind `address`, if there is one. - /// - /// One `metadata` call, no read. Used where an indexed name has to be checked against - /// what a caller is offering before that offer is turned away. - #[must_use] - pub fn stored_len(&self, address: &XorName) -> Option { - std::fs::metadata(self.chunk_path(address)) - .ok() - .filter(std::fs::Metadata::is_file) - .and_then(|m| usize::try_from(m.len()).ok()) - } - - /// Whether the file already stored under `address` really hashes to it. - async fn stored_bytes_match(&self, address: &XorName) -> StoredBytes { - match self.get_raw(address).await { - Ok(Some(bytes)) if crate::client::compute_address(&bytes) == *address => { - // A read that hashed. It settles both questions. - self.clear_suspect(address); - self.clear_known_wrong(address); - StoredBytes::Good - } - Ok(Some(_)) => { - self.mark_known_wrong(address); - StoredBytes::Wrong - } - Ok(None) => StoredBytes::Absent, - // NOT the same as wrong. A file that could not be read this once may be - // perfectly good, and off Unix replacing it means opening it with `truncate`, - // which would destroy a healthy sole copy on the strength of a transient - // fault. Say so and let the caller leave it alone. - Err(e) => { - debug!("Could not read {} to check it: {e}", hex::encode(address)); - self.mark_suspect(address); - StoredBytes::Unreadable - } - } - } - - /// Replace the file behind an address with known-good bytes, atomically. - /// - /// Unlike [`Self::put`], this deliberately publishes **over** an existing name. It - /// exists for one caller: repairing a file whose bytes no longer hash to their own - /// name, from a copy held elsewhere, before that copy is destroyed. Doing it as - /// delete-then-put would leave a window where the only remaining copy is the one - /// about to be deleted, and any failure in that window is unrecoverable. - /// - /// # Errors - /// - /// Returns [`Error::Storage`] if `content` does not hash to `address`, or the write - /// fails. The old file is left untouched on every error path. - pub async fn repair(&self, address: &XorName, content: &[u8]) -> Result<()> { - // The same ceiling `put` enforces. Without it a repair can install bytes the read - // path will refuse for ever, which is a chunk that verifies as present and can - // never be served. - if content.len() > MAX_CHUNK_SIZE { - return Err(Error::Storage(format!( - "Refusing to repair {} with {} bytes, over the {MAX_CHUNK_SIZE} byte \ - maximum", - hex::encode(address), - content.len() - ))); - } - let computed = crate::client::compute_address(content); - if computed != *address { - return Err(Error::Storage(format!( - "Refusing to repair {} with content that hashes to {}", - hex::encode(address), - hex::encode(computed) - ))); - } - // The replacement exists alongside the original until the rename, so the room for - // it has to be there first. Reserved rather than merely checked: a plain check - // passes against a cached measurement, so concurrent repairs and PUTs can each be - // admitted against the same headroom and cross the reserve together. - // Moved into the work below, so it is released when the write finishes rather - // than when its caller stops waiting. A caller that goes away otherwise frees - // room that the detached write is still about to consume. - let reservation = self.capacity.reserve(content.len() as u64)?; - - let shard = self.chunks_dir.join(shard_name(address)); - let final_path = shard.join(hex::encode(address)); - let temp_path = shard.join(self.next_temp_name()); - let payload = content.to_vec(); - let lanes = Arc::clone(&self.write_lanes); - let index = Arc::clone(&self.index); - let shards_present = Arc::clone(&self.shards_present); - let chunks_dir = self.chunks_dir.clone(); - let lane = shard_index(address); - let key = *address; - let in_flight = self.begin_write(address); - let lease = Arc::clone(&self.lock); - let capacity = Arc::clone(&self.capacity); - let suspect = Arc::clone(&self.suspect); - let known_wrong = Arc::clone(&self.known_wrong); - - self.blocking_tracker - .spawn_blocking(move || -> Result<()> { - let _in_flight = in_flight; - let _lease = lease; - let _reservation = reservation; - let _lane = lanes.get(lane).map(parking_lot::Mutex::lock); - ensure_shard_dir(&chunks_dir, &shard, lane, &shards_present)?; - write_and_replace(&temp_path, &final_path, &payload, &shard)?; - index.write().insert(key); - // Settled here rather than after the await. The replacement has landed - // and hashes to its own name, so nothing is wrong with this chunk any - // more; a caller that stopped waiting would otherwise leave a healthy - // file excluded from everything the node claims to hold, and the - // measurement believing the store is a chunk smaller than it is. - suspect.write().remove(&key); - known_wrong.write().remove(&key); - capacity.invalidate(); - Ok(()) - }) - .await - .map_err(|e| Error::Storage(format!("Chunk store repair task failed: {e}")))??; - - // Everything the success means was recorded by the work that succeeded: the - // reservation released, the marks cleared, the measurement thrown away. Released - // rather than committed because a repair is not a new chunk, and the measurement - // discarded rather than adjusted because the file it replaced may have been - // shorter, which is exactly the case a repair fixes. - debug!("Repaired chunk {}", hex::encode(address)); - Ok(()) - } - - /// Retrieve a chunk, verifying it against its address when configured to. - /// - /// # Errors - /// - /// Returns [`Error::Storage`] on an I/O failure, or when verification fails. A - /// chunk whose bytes do not hash to its name is removed and dropped from the index - /// before the error is returned, so it leaves `all_keys()` and ordinary replication - /// repairs it. - pub async fn get(&self, address: &XorName) -> Result>> { - let Some(content) = self.read_file(address).await? else { - trace!("Chunk {} not found", hex::encode(address)); - return Ok(None); - }; - - if self.config.verify_on_read { - let computed = crate::client::compute_address(&content); - if computed != *address { - { - let mut stats = self.stats.write(); - stats.verification_failures = stats.verification_failures.saturating_add(1); - } - warn!( - "Chunk verification failed: expected {}, computed {}", - hex::encode(address), - hex::encode(computed) - ); - // Said before it is acted on. Removing the file can fail or be cancelled, - // and a chunk proven wrong that goes on looking healthy is one the node - // keeps committing to and, worse, one a cached pre-retirement pass still - // covers: the legacy copy that would repair it gets deleted. - self.mark_known_wrong(address); - self.quarantine_corrupt(address).await; - return Err(Error::Storage(format!( - "Chunk verification failed for {}", - hex::encode(address) - ))); - } - } - - if self.config.verify_on_read { - // The bytes hashed to their name. Whatever this store thought was wrong with - // them is not wrong with them, and a mark that outlives the fault it - // describes means the node can serve a chunk it will not claim, commit or - // offer. - self.clear_known_wrong(address); - } - - let len = content.len() as u64; - { - let mut stats = self.stats.write(); - stats.chunks_retrieved = stats.chunks_retrieved.saturating_add(1); - stats.bytes_retrieved = stats.bytes_retrieved.saturating_add(len); - } - debug!("Retrieved chunk {} ({len} bytes)", hex::encode(address)); - Ok(Some(content)) - } - - /// Retrieve raw chunk bytes without content-address verification. - /// - /// # Errors - /// - /// Returns [`Error::Storage`] on an I/O failure. - pub async fn get_raw(&self, address: &XorName) -> Result>> { - self.read_file(address).await - } - - /// Check whether a chunk is stored. - /// - /// An in-memory lookup: no syscall, no I/O. - /// - /// # Errors - /// - /// Never fails. The signature keeps the shape the LMDB store had, because callers - /// treat the error as "assume absent". - pub fn exists(&self, address: &XorName) -> Result { - if self.is_unservable(address) { - return Ok(false); - } - Ok(self.is_indexed(address)) - } - - /// Is this chunk one the node must not answer for? - #[must_use] - fn is_unservable(&self, address: &XorName) -> bool { - self.suspect.read().contains(address) || self.known_wrong.read().contains(address) - } - - /// Is this chunk in the index, whether or not it can currently be read? - /// - /// The physical question, as against [`Self::exists`]'s question about what the node - /// is willing to claim. The migration must ask this one: a suspect chunk is still a - /// file this store has, and treating it as absent would put the key in the legacy-only - /// set, from where the union view advertises it again — a key the node claims through - /// one view and cannot serve through either. - #[must_use] - pub fn is_indexed(&self, address: &XorName) -> bool { - self.index.read().contains(address) - } - - /// Delete a chunk, returning whether it was present. - /// - /// `unlink` returns the blocks to the filesystem immediately. That is the whole - /// point of this store: no free list, no compaction, no free space required to - /// reclaim space. - /// - /// # Errors - /// - /// Returns [`Error::Storage`] if the file exists but cannot be removed. The index - /// keeps the key in that case, because the bytes are still on disk. - pub async fn delete(&self, address: &XorName) -> Result { - let path = self.chunk_path(address); - let lanes = Arc::clone(&self.write_lanes); - let index = Arc::clone(&self.index); - let lane = shard_index(address); - let key = *address; - // Carried into the closure for the reason `put`, `repair` and the startup scan - // carry it: this work outlives the future that started it, so a cancelled caller - // that drops the last `FileStore` would otherwise release the directory to another - // process while an unlink is still queued against it. Deleting is the operation - // where that matters most. - let lease = Arc::clone(&self.lock); - - let (existed, freed) = self - .blocking_tracker - .spawn_blocking(move || -> Result<(bool, u64)> { - let _lease = lease; - let _lane = lanes.get(lane).map(parking_lot::Mutex::lock); - let len = std::fs::metadata(&path).map_or(0, |m| m.len()); - let removed = match std::fs::remove_file(&path) { - Ok(()) => { - // Without this a crash can resurrect the entry on ext4, XFS, - // btrfs and APFS: the unlink is in the page cache, the directory - // is not. - if let Some(shard) = path.parent() { - fsync_dir_best_effort(shard); - } - true - } - // Already gone: the index was stale. Still a successful delete as - // far as the caller is concerned. - Err(e) if e.kind() == ErrorKind::NotFound => false, - Err(e) => { - return Err(Error::Storage(format!( - "Failed to delete chunk file {}: {e}", - path.display() - ))) - } - }; - // Index only after the filesystem operation has succeeded. On the error - // path above the entry stays, because the bytes are still on disk. - let was_indexed = index.write().remove(&key); - Ok((removed || was_indexed, if removed { len } else { 0 })) - }) - .await - .map_err(|e| Error::Storage(format!("Chunk store delete task failed: {e}")))??; - - if freed > 0 { - self.capacity.record_removed(freed); - debug!("Deleted chunk {}", hex::encode(address)); - } - Ok(existed) - } - - /// Return every stored key, in ascending order. - /// - /// The order is a correctness requirement, not a convenience: the commitment - /// builder truncates the responsible subset with `take(cap)` before the Merkle tree - /// sorts it, so an unstable order would make the node's published commitment depend - /// on iteration luck. - /// - /// # Errors - /// - /// Never fails. The signature matches the LMDB store's. - // Async without awaiting anything, deliberately: the whole point of this store is - // that the key set is already in memory. Callers are spread across the replication - // engine and cannot all be de-async'd in this change. - // - // Two lint names because they were renamed between toolchains, and `unknown_lints` - // so whichever one the compiler in use has never heard of stays quiet. - #[allow(unknown_lints)] - #[allow(clippy::unused_async, clippy::unused_async_trait_impl)] - pub async fn all_keys(&self) -> Result> { - // Copied out first so neither lock is held while the other is taken, and so the - // usual case, where nothing is suspect, costs one clone of an empty set. - let mut unservable: HashSet = self.suspect.read().clone(); - unservable.extend(self.known_wrong.read().iter().copied()); - let keys = self.index.read().clone(); - if unservable.is_empty() { - return Ok(keys.into_iter().collect()); - } - Ok(keys - .into_iter() - .filter(|key| !unservable.contains(key)) - .collect()) - } - - /// Stop answering for a chunk this store could not read. - /// - /// The file stays. It may be perfectly good and unreadable only for the moment, and - /// deleting it, or dropping it from the index, is how a chunk ends up in neither this - /// store's view nor the legacy one, which is what retirement destroys. - /// - /// What does change is what the node says about it. A chunk it cannot read is one it - /// cannot serve, and claiming it anyway puts the key in signed commitments, answers - /// presence probes with a yes, suppresses the replication that would repair it, and - /// earns a penalty at the next commitment-bound audit. Those penalties are not - /// suspended. - fn mark_suspect(&self, address: &XorName) { - if self.suspect.write().insert(*address) { - self.note_health_changed(); - warn!( - "Chunk {} is on disk but could not be read; this node stops answering for \ - it until a read succeeds", - hex::encode(address) - ); - } - } - - /// What the store's health looked like at this moment. - /// - /// Compare a value taken before a long-running check with one taken after, or after - /// taking a lock: different means a chunk stopped being servable in between and any - /// conclusion drawn from that check is out of date. - #[must_use] - pub fn health_generation(&self) -> u64 { - self.health.load(std::sync::atomic::Ordering::Acquire) - } - - /// Record that a chunk stopped being servable. - fn note_health_changed(&self) { - self.health - .fetch_add(1, std::sync::atomic::Ordering::AcqRel); - } - - /// Stop answering for a chunk a read has proven wrong. - /// - /// Unlike a chunk that merely could not be read, a later read does not clear this. - /// The bytes are wrong, and reading them again says the same thing; only replacing - /// them or removing them settles it. Bumping health matters as much as the suppression: a chunk that - /// has become unservable since the last pre-retirement pass must invalidate that pass, - /// or a repair that fails leaves the node deleting the copy it would have repaired - /// from. - /// - /// For callers outside this module that have proven it themselves. - pub fn note_known_wrong(&self, address: &XorName) { - self.mark_known_wrong(address); - } - - /// Stop answering for a chunk a read has proven wrong. - fn mark_known_wrong(&self, address: &XorName) { - if self.known_wrong.write().insert(*address) { - self.note_health_changed(); - warn!( - "Chunk {} does not match its name; this node stops answering for it until \ - it is repaired or removed", - hex::encode(address) - ); - } - } - - /// A caller outside this module has proven the stored bytes are right. - pub fn note_bytes_proven_good(&self, address: &XorName) { - self.clear_known_wrong(address); - self.clear_suspect(address); - } - - /// Answer for a chunk again, after it has been replaced or removed. - fn clear_known_wrong(&self, address: &XorName) { - self.known_wrong.write().remove(address); - } - - /// Answer for a chunk again, after a read that worked. - fn clear_suspect(&self, address: &XorName) { - if !self.suspect.read().contains(address) { - return; - } - if self.suspect.write().remove(address) { - info!( - "Chunk {} could be read again; this node answers for it once more", - hex::encode(address) - ); - } - } - - /// Number of chunks currently stored. - /// - /// The physical count: every name in the index, including chunks the node has stopped - /// answering for because a read found them wrong or could not read them at all. It is - /// deliberately not the same number as `all_keys().len()`, which is what the node is - /// willing to claim and so leaves those out. - /// - /// Anything asking "how much is on this disk" wants this one, and that is what its - /// callers ask: the migration's progress, the storage stats, and the size an audit is - /// built for. Anything asking "what will this node answer for" wants `all_keys`. - /// Quietly filtering this one would move all three of those without saying so, which - /// is why the difference is written down here rather than removed. - /// - /// # Errors - /// - /// Never fails. The signature matches the LMDB store's. - pub fn current_chunks(&self) -> Result { - Ok(self.index.read().len() as u64) - } - - /// Operation statistics, with the live chunk count filled in. - #[must_use] - pub fn stats(&self) -> StorageStats { - let mut stats = self.stats.read().clone(); - stats.current_chunks = self.index.read().len() as u64; - stats - } - - /// The node root directory this store was configured with. - #[must_use] - pub fn root_dir(&self) -> &Path { - &self.config.root_dir - } - - /// The directory holding the shard tree. - #[must_use] - pub fn chunks_dir(&self) -> &Path { - &self.chunks_dir - } - - /// Reject work early when the disk cannot take another chunk at all. - /// - /// # Errors - /// - /// Returns [`Error::Storage`] when free space is below the configured reserve. - pub fn check_capacity(&self) -> Result<()> { - self.capacity.check(0) - } - - /// Three-way answer to "can this store take a write right now". - /// - /// Kept distinct from [`Self::check_capacity`] because a failed free-space query and a - /// genuinely full disk are not the same thing, and the replication verification cycle - /// depends on the difference: a full disk is a standing condition worth minutes of - /// backoff, while a `statvfs` that failed says nothing about available space and may - /// well succeed on the next pass. - #[must_use] - pub fn capacity_verdict(&self) -> crate::storage::CapacityVerdict { - match self.capacity.measure_available() { - Some(available) if available < self.capacity.reserve => { - crate::storage::CapacityVerdict::Full - } - Some(_) => crate::storage::CapacityVerdict::Writable, - None => crate::storage::CapacityVerdict::Unknown, - } - } - - /// Credit `len` bytes that were deleted outside this store back to the - /// cached measurement. The pointer store keeps its own files on this disk. - pub(crate) fn release_bytes(&self, len: u64) { - self.capacity.record_removed(len); - } - - /// Reject work early when the disk cannot take `bytes` more. - /// - /// # Errors - /// - /// Returns [`Error::Storage`] when the write would not fit above the reserve. - pub fn check_capacity_for(&self, bytes: u64) -> Result<()> { - self.capacity.check(bytes) - } - - /// Charge `bytes` against the disk before writing them, releasing the charge if the - /// write does not happen. - /// - /// For stores that share this disk but keep their own files — the pointer store does. - /// They must not check and then write: that is the race [`CapacityGuard::reserve`] - /// exists to close. - pub(crate) fn reserve_bytes(&self, bytes: u64) -> Result { - self.capacity.reserve(bytes) - } - - /// `(written_since, in_flight)` from the capacity guard. Tests only: it is - /// how a released charge is told apart from a stranded one. - #[cfg(test)] - pub(crate) fn capacity_counters(&self) -> (u64, u64) { - self.capacity.counters() - } - - /// What a payload of `bytes` actually costs the disk, rounded to the - /// allocation unit. Tests only, so they can assert exact counter deltas - /// rather than "it went up" — which a charge of one allocation unit would - /// satisfy for several payloads at once. - #[cfg(test)] - pub(crate) fn capacity_charge_for(bytes: u64) -> u64 { - CapacityGuard::charge(bytes) - } - - /// Force the next capacity question to re-measure the filesystem. - /// - /// Called after the legacy environment is removed, because that is a step change in - /// free space that the short-lived cache would otherwise hide for a few seconds. - pub fn invalidate_capacity_cache(&self) { - self.capacity.invalidate(); - } - - /// Test-only handle to the put gate. - /// - /// Hold the write half to park the next write inside its blocking closure, for - /// example to prove that shutdown waits for a write whose awaiter was dropped. - #[cfg(any(test, feature = "test-utils"))] - #[must_use] - pub fn test_put_gate(&self) -> Arc> { - Arc::clone(&self.test_put_gate) - } - - /// Register a write of `address` and hand back the token that clears it. - /// - /// The token must be moved into the blocking closure that does the work, so the entry - /// is cleared by the thread that finishes rather than by a caller that may be gone. - fn begin_write(&self, address: &XorName) -> WriteInFlight { - *self.writing.lock().entry(*address).or_insert(0) += 1; - WriteInFlight { - writing: Arc::clone(&self.writing), - finished: Arc::clone(&self.write_finished), - address: *address, - } - } - - /// Wait until nothing is part-way through writing `address`. - /// - /// For callers that must be last: a delete whose key still has a write in flight - /// would be undone by that write landing afterwards. - pub async fn wait_for_write(&self, address: &XorName) { - loop { - // Registered before the check, so a clear between the two is not missed. - let waiting = self.write_finished.notified(); - if !self.writing.lock().contains_key(address) { - return; - } - waiting.await; - } - } - - /// How many blocking tasks this store currently has in flight. Tests only. - /// - /// Lets a test wait for work to have actually started rather than guessing at a - /// delay, which is the difference between a test that proves something and one that - /// passes because the machine was quick. - #[cfg(any(test, feature = "test-utils"))] - #[must_use] - pub(crate) fn tasks_in_flight(&self) -> usize { - self.blocking_tracker.len() - } - - /// Wait until every blocking task this store spawned has finished. - /// - /// Dropping the awaiting future does not cancel a `spawn_blocking` closure, so - /// shutdown has to wait for the closure itself. - pub async fn wait_idle(&self) { - self.blocking_tracker.close(); - self.blocking_tracker.wait().await; - self.blocking_tracker.reopen(); - } - - /// Absolute path of a chunk file. - fn chunk_path(&self, address: &XorName) -> PathBuf { - self.chunks_dir - .join(shard_name(address)) - .join(hex::encode(address)) - } - - /// A temp name unique to this store instance, and distinguishable from a chunk name. - /// - /// The nonce matters: two `FileStore`s on one root in one process share a PID, and a - /// recycled PID collides with an age-gated leftover. Either way `create_new` would - /// fail and surface as a spurious write error. - fn next_temp_name(&self) -> String { - let seq = self.temp_seq.fetch_add(1, Ordering::Relaxed); - format!( - "{TEMP_PREFIX}{}.{:08x}.{seq}", - std::process::id(), - self.nonce - ) - } - - /// Read a chunk file, dropping the index entry if the file has vanished. - async fn read_file(&self, address: &XorName) -> Result>> { - let path = self.chunk_path(address); - let read = self - .blocking_tracker - .spawn_blocking(move || -> Result>> { - match open_regular(&path) { - Ok(Some(f)) => read_bounded(f, &path).map(Some), - Ok(None) => Ok(None), - Err(e) => Err(e), - } - }) - .await - .map_err(|e| Error::Storage(format!("Chunk store read task failed: {e}")))?; - - // Every read decides the question, not only the ones that were checking. A read - // that failed means this chunk cannot be served, whoever asked; a read that - // worked means it can be, whoever asked. Doing this anywhere else leaves a key - // stuck unadvertised after the fault has cleared, or advertised after it has not. - let read = match read { - Ok(read) => { - self.clear_suspect(address); - read - } - Err(e) => { - self.mark_suspect(address); - return Err(e); - } - }; - - if read.is_none() && self.forget_if_absent(address).await { - // The file went away underneath us. Stop advertising the key so the close - // group notices the shortfall and replication puts it back. - warn!( - "Chunk {} is indexed but its file is missing; dropped from the index so \ - replication can repair it", - hex::encode(address) - ); - } - Ok(read) - } - - /// Drop an index entry whose file is genuinely gone. - /// - /// Re-checks under the address's write lane, so a chunk republished between the - /// failing read and this call keeps its entry. - async fn forget_if_absent(&self, address: &XorName) -> bool { - // Not suspect any more: it is not unreadable, it is not there. - self.clear_suspect(address); - let path = self.chunk_path(address); - let lanes = Arc::clone(&self.write_lanes); - let index = Arc::clone(&self.index); - let lane = shard_index(address); - let key = *address; - // The bump happens inside the closure, with the mutation it describes. The - // closure runs to completion on its own thread whether or not anyone is still - // awaiting it, so bumping after the await is skipped entirely when a shutdown - // drops the caller — and the index change it was meant to announce still lands. - // A cached pre-retirement proof would then stay valid over a store that had - // quietly lost a chunk. - let health = Arc::clone(&self.health); - self.blocking_tracker - .spawn_blocking(move || { - let _lane = lanes.get(lane).map(parking_lot::Mutex::lock); - if path.exists() { - return false; - } - let forgotten = index.write().remove(&key); - if forgotten { - health.fetch_add(1, std::sync::atomic::Ordering::AcqRel); - } - forgotten - }) - .await - .unwrap_or(false) - } - - /// Remove a chunk whose bytes do not match its name, and stop advertising it. - /// - /// Re-reads and re-verifies under the address's write lane first. A read that failed - /// verification is rare enough that paying for one extra read is worth never - /// discarding a chunk that a concurrent write had already repaired. - async fn quarantine_corrupt(&self, address: &XorName) { - let path = self.chunk_path(address); - let lanes = Arc::clone(&self.write_lanes); - let index = Arc::clone(&self.index); - let lane = shard_index(address); - let key = *address; - // For the reason given on `forget_if_absent`: this closure outlives its awaiter, - // and the change it makes has to be announced by the same thread that makes it. - let health = Arc::clone(&self.health); - // And the store-lock lease, for the reason `put`, `repair`, `delete` and the - // startup scan carry it: this closure outlives its awaiter, so without it a - // cancelled verification whose caller dropped the last `FileStore` would unlink - // inside a directory a second process had already been handed. - let lease = Arc::clone(&self.lock); - let outcome = - self.blocking_tracker - .spawn_blocking(move || -> std::io::Result { - let _lease = lease; - let _lane = lanes.get(lane).map(parking_lot::Mutex::lock); - // Nothing is thrown away without proof. A re-read that fails says the - // question could not be answered this time, not that the bytes are wrong, - // and a repair may have published a good copy since the read that brought - // us here. Treating either as corruption deletes a chunk this node has. - let buf = match open_regular(&path) { - Ok(Some(f)) => read_bounded(f, &path) - .map_err(|e| std::io::Error::other(e.to_string()))?, - Ok(None) => { - index.write().remove(&key); - health.fetch_add(1, std::sync::atomic::Ordering::AcqRel); - return Ok(true); - } - Err(e) => return Err(std::io::Error::other(e.to_string())), - }; - if crate::client::compute_address(&buf) == key { - // Repaired between the failing read and now. Leave it alone. - return Ok(false); - } - std::fs::remove_file(&path)?; - // The same flush the ordinary delete does, for the same reason: an - // unlink that has not reached the directory can be undone by a power - // loss, and here the entry that comes back is one this node has proven - // wrong. The startup scan would re-index it by name, and the - // known-wrong mark that would otherwise hold it back lives only in - // memory and does not survive the restart, so the node would go back to - // claiming and committing to a chunk it already knows is bad. - if let Some(shard) = path.parent() { - fsync_dir_best_effort(shard); - } - index.write().remove(&key); - health.fetch_add(1, std::sync::atomic::Ordering::AcqRel); - Ok(true) - }) - .await; - match outcome { - Ok(Ok(true)) => { - self.clear_known_wrong(address); - self.clear_suspect(address); - warn!( - "Removed corrupt chunk file {}; replication will repair it", - hex::encode(address) - ); - } - Ok(Ok(false)) => { - // The re-read hashed and matched: a repair landed between the failing - // read and this one. - self.clear_known_wrong(address); - self.clear_suspect(address); - debug!( - "Chunk {} verified on re-read; leaving it in place", - hex::encode(address) - ); - } - // Still indexed, so it must not still be claimed: the read that brought us - // here proved the bytes wrong, and the node would otherwise go on committing - // to a chunk it knows it cannot serve. - Ok(Err(e)) => { - self.mark_suspect(address); - warn!( - "Corrupt chunk {} could not be removed: {e}. It stays on disk, and \ - this node stops answering for it.", - hex::encode(address) - ); - } - Err(e) => { - self.mark_suspect(address); - warn!("Corrupt-chunk removal task failed: {e}"); - } - } - } -} - -// ──────────────────────────────────────────────────────────────────────────── -// Free functions -// ──────────────────────────────────────────────────────────────────────────── - -/// Create the destination shard directory if this store has not seen it yet. -/// -/// A newly created directory entry is only durable once its parent is flushed; without -/// that a crash could take the directory and the chunk inside it together. -fn ensure_shard_dir( - chunks_dir: &Path, - dir: &Path, - shard: usize, - present: &parking_lot::Mutex<[bool; SHARD_COUNT]>, -) -> Result<()> { - if present.lock().get(shard).copied().unwrap_or(false) { - return Ok(()); - } - std::fs::create_dir_all(dir).map_err(|e| { - Error::Storage(format!( - "Failed to create shard directory {}: {e}", - dir.display() - )) - })?; - // Load-bearing, like the flush that publishes a chunk into this directory. Until the - // parent is flushed the shard's own entry can be lost, and losing it loses every chunk - // inside it. Reporting the shard present anyway would let the very first chunk written - // into it count as durably stored. - fsync_dir(chunks_dir).map_err(|e| { - Error::Storage(format!( - "Created shard directory {} but could not flush {}: {e}. Not marking the shard \ - usable, because a directory that is not durable cannot hold a chunk that is.", - dir.display(), - chunks_dir.display() - )) - })?; - if let Some(slot) = present.lock().get_mut(shard) { - *slot = true; - } - Ok(()) -} - -/// Shard directory index for an address: its last byte. -fn shard_index(address: &XorName) -> usize { - address.last().copied().unwrap_or(0) as usize -} - -/// Shard directory name for an address: the last two characters of its hex form. -fn shard_name(address: &XorName) -> String { - format!("{:02x}", shard_index(address)) -} - -/// True for a string of hex digits in either case. -fn is_hex_any_case(s: &str) -> bool { - !s.is_empty() && s.bytes().all(|b| b.is_ascii_hexdigit()) -} - -/// Move an entry aside under a name that can never be read as a chunk. -fn quarantine_entry(path: &Path) { - let aside = path.with_extension("not-a-chunk"); - match std::fs::rename(path, &aside) { - Ok(()) => warn!( - "Chunk store: moved {} aside to {}; a name that differs from a chunk name only \ - by case collides with it on Windows and macOS", - path.display(), - aside.display() - ), - Err(e) => warn!( - "Chunk store: {} collides with a chunk name by case folding and could not be \ - moved aside: {e}. Rename or delete it.", - path.display() - ), - } -} - -/// True for a string of lowercase hex digits only. -fn is_lower_hex(s: &str) -> bool { - !s.is_empty() && s.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')) -} - -/// Decode a filename back into the address it names, or `None` if it is not one. -/// -/// Rejects uppercase deliberately. On a case-folding filesystem (NTFS, default APFS) -/// accepting both cases would let one file answer to two index entries. -fn decode_chunk_name(name: &str) -> Option { - if name.len() != CHUNK_NAME_LEN || !is_lower_hex(name) { - return None; - } - let bytes = hex::decode(name).ok()?; - XorName::try_from(bytes.as_slice()).ok() -} - -/// Flush a directory and report whether it worked, for callers outside this module. -/// -/// For the one caller whose next step is destructive: retirement moves the legacy -/// environment aside and then deletes it under its new name, so if the rename has not -/// reached the disk when the delete lands, a power loss brings the environment back under -/// its old name with its contents gone. -/// -/// # Errors -/// -/// Returns the underlying I/O error. Off Unix there is no way to flush a directory through -/// the standard library, so this reports success without being able to promise anything. -pub fn fsync_path(path: &Path) -> std::io::Result<()> { - fsync_dir(path) -} - -/// Flush a directory so a rename or creation inside it survives power loss. -/// -/// Best effort by design. Linux and XFS require it, macOS accepts it with undocumented -/// effect, and Windows offers no way to do it at all through the standard library. The -/// content is content-addressed and re-replicable, so a lost directory entry costs a -/// refetch rather than data. Pretending otherwise in the code would be dishonest. -#[cfg(unix)] -fn fsync_dir_best_effort(path: &Path) { - if let Err(e) = fsync_dir(path) { - debug!("Directory flush of {} failed: {e}", path.display()); - } -} - -/// Flush a directory, reporting whether it worked. -/// -/// Used where the answer is load-bearing: a chunk copied out of the legacy store is only -/// durable once its directory entry is, and that copy is what permits the legacy store to -/// be deleted. -#[cfg(unix)] -fn fsync_dir(path: &Path) -> std::io::Result<()> { - File::open(path)?.sync_all() -} - -/// Off Unix there is no way to flush a directory through the standard library, so this -/// reports success without being able to promise anything. -/// -/// That is why the publish path off Unix does not use a rename at all: it creates the -/// chunk under its final name and flushes the file, which Microsoft documents as flushing -/// the creation metadata with it. Directory creation has no equivalent, so the guarantee -/// there rests on the pre-retirement pass, which re-reads every chunk before the legacy -/// store is deleted, and on the operator gate that keeps retirement off a platform until -/// forced power loss has been shown to hold old-or-new on it. -/// -/// Returns a `Result` so the callers that must handle a flush failure on Unix read the -/// same on every platform. -#[cfg(not(unix))] -#[allow(clippy::unnecessary_wraps)] -fn fsync_dir(_path: &Path) -> std::io::Result<()> { - Ok(()) -} - -/// No-op on platforms with no way to flush a directory handle. -#[cfg(not(unix))] -fn fsync_dir_best_effort(_path: &Path) {} - -/// Warn if the deepest chunk path this store can produce is close to `MAX_PATH`. -#[cfg(windows)] -fn check_path_budget(chunks_dir: &Path) { - // Measured absolute, because that is what the filesystem sees. A relative root is the - // case that still fails hard at MAX_PATH, since the standard library's long-path - // handling only applies to paths it resolves as absolute. - let absolute = if chunks_dir.is_absolute() { - chunks_dir.to_path_buf() - } else { - std::env::current_dir() - .map_or_else(|_| chunks_dir.to_path_buf(), |cwd| cwd.join(chunks_dir)) - }; - // `{chunks_dir}\{xy}\{64 hex}` — two separators, two shard characters, 64 name - // characters. - let deepest = absolute.as_os_str().len() + 1 + 2 + 1 + CHUNK_NAME_LEN; - if deepest > WINDOWS_PATH_WARN_LEN { - warn!( - "Chunk file paths will be {deepest} characters, close to the {} character \ - Windows limit. Move the node root closer to the drive letter if writes start \ - failing.", - WINDOWS_PATH_WARN_LEN - ); - } -} - -/// No-op where path length is not a practical constraint. -#[cfg(not(windows))] -fn check_path_budget(_chunks_dir: &Path) {} - -/// Write `bytes` to `path` durably, for small metadata files outside the shard tree. -/// -/// # Errors -/// -/// Returns [`Error::Storage`] if the file cannot be written or published. -pub fn write_file_durably(path: &Path, bytes: &[u8]) -> Result<()> { - write_file_atomic(path, bytes) -} - -/// Write `bytes` to `path` so a reader sees either the old content or the new. -/// Is this the exact name [`write_file_atomic`] gives its temporaries? -/// -/// `.tmp..<8 hex>.marker`, with both middle parts checked. Matching on the prefix and -/// suffix alone would also take `.tmp.operator-notes.marker`, and this runs over a -/// directory holding a node's data, so what it removes is not a place to be approximate. -fn is_marker_temp_name(name: &str) -> bool { - let Some(rest) = name.strip_prefix(TEMP_PREFIX) else { - return false; - }; - let Some(rest) = rest.strip_suffix(".marker") else { - return false; - }; - let mut parts = rest.split('.'); - let (Some(pid), Some(nonce), None) = (parts.next(), parts.next(), parts.next()) else { - return false; - }; - !pid.is_empty() - && pid.bytes().all(|b| b.is_ascii_digit()) - && nonce.len() == 8 - && nonce.bytes().all(|b| b.is_ascii_hexdigit()) -} - -/// Remove marker temporaries a previous run left beside `path`. -/// -/// [`write_file_atomic`] writes its temporary next to its target. For the layout marker -/// that is inside `chunks/`, which the startup scan sweeps; for the migration marker it is -/// the node root, which nothing sweeps, so a crash between the write and the rename leaves -/// one there for the life of the node. Each is a few hundred bytes, so this is inodes -/// rather than capacity, but nothing else was ever going to remove them. -/// -/// Only the exact shape this module writes, and only files: a name has to carry the temp -/// prefix and the marker suffix. Anything broader would be this function deciding what -/// else in a node's root directory is rubbish, which is not its business. -/// -/// Best effort throughout. Failing to tidy up is not a reason to refuse to start, and the -/// caller takes the store lock before this runs, so there is no other process whose live -/// temporary this could take. -pub(crate) fn sweep_marker_temps(dir: &Path) { - let Ok(entries) = std::fs::read_dir(dir) else { - return; - }; - for entry in entries.flatten() { - let name = entry.file_name(); - let Some(name) = name.to_str() else { - continue; - }; - if !is_marker_temp_name(name) { - continue; - } - if !entry.file_type().is_ok_and(|kind| kind.is_file()) { - continue; - } - match std::fs::remove_file(entry.path()) { - Ok(()) => debug!( - "Swept a leftover marker temporary {}", - entry.path().display() - ), - Err(e) => debug!("Could not sweep {}: {e}", entry.path().display()), - } - } -} - -fn write_file_atomic(path: &Path, bytes: &[u8]) -> Result<()> { - let Some(dir) = path.parent() else { - return Err(Error::Storage(format!( - "Refusing to write {} — it has no parent directory", - path.display() - ))); - }; - let temp = dir.join(format!( - "{TEMP_PREFIX}{}.{:08x}.marker", - std::process::id(), - rand::random::() - )); - write_temp(&temp, bytes)?; - // Through the retry, because these small files (the layout marker, the migration - // state) are rewritten while the node runs, and on Windows a scanner holding a handle - // for a few milliseconds turns an ordinary rewrite into a hard failure. - rename_with_retry(&temp, path).map_err(|e| { - let _ = std::fs::remove_file(&temp); - Error::Storage(format!("Failed to publish {}: {e}", path.display())) - })?; - fsync_dir_best_effort(dir); - Ok(()) -} - -/// Read the layout marker, writing the current one if the store is new. -fn read_or_write_layout(chunks_dir: &Path) -> Result { - let path = chunks_dir.join(LAYOUT_FILE_NAME); - match read_small_file(&path) { - Ok(bytes) => serde_json::from_slice(&bytes).map_err(|e| { - Error::Storage(format!( - "Chunk store layout marker {} is unreadable: {e}. Refusing to open rather \ - than guess the layout.", - path.display() - )) - }), - Err(e) if e.kind() == ErrorKind::NotFound => { - if store_has_entries(chunks_dir) { - warn!( - "Chunk store at {} has data but no layout marker. Adopting it under \ - the current scheme, which is the only one this build implements. If \ - it was written by a build with a different layout its chunks will \ - appear to be missing.", - chunks_dir.display() - ); - } - let layout = StoreLayout::default(); - let bytes = serde_json::to_vec_pretty(&layout) - .map_err(|e| Error::Storage(format!("Failed to encode chunk store layout: {e}")))?; - write_file_atomic(&path, &bytes)?; - debug!("Wrote chunk store layout marker to {}", path.display()); - Ok(layout) - } - Err(e) => Err(Error::Storage(format!( - "Failed to read chunk store layout marker {}: {e}", - path.display() - ))), - } -} - -/// Whether the store directory already holds at least one shard. -fn store_has_entries(chunks_dir: &Path) -> bool { - let Ok(entries) = std::fs::read_dir(chunks_dir) else { - return false; - }; - entries.filter_map(std::result::Result::ok).any(|e| { - e.file_name() - .to_str() - .is_some_and(|n| n.len() == 2 && is_lower_hex(n)) - }) -} - -/// Largest a metadata marker may be before it is treated as corrupt. -const MAX_MARKER_BYTES: u64 = 64 * 1024; - -/// Read a small metadata file, refusing an implausibly large one. -/// -/// The chunk path is bounded for exactly this reason; the markers live in the same data -/// directory and deserve the same ceiling. -/// -/// # Errors -/// -/// Returns an I/O error, including `NotFound`, so callers can distinguish "no marker yet". -pub fn read_small_file(path: &Path) -> std::io::Result> { - let file = File::open(path)?; - let mut bytes = Vec::new(); - let read = file.take(MAX_MARKER_BYTES + 1).read_to_end(&mut bytes)?; - if read as u64 > MAX_MARKER_BYTES { - return Err(std::io::Error::other(format!( - "{} is larger than the {MAX_MARKER_BYTES} byte limit for a marker file", - path.display() - ))); - } - Ok(bytes) -} - -/// Take the store lock, or refuse to open the store. -/// -/// Both failures are refusals, deliberately. Unlike LMDB, which was genuinely -/// multi-process safe, two of these stores on one directory keep independent in-memory -/// indices, independent views of what is in flight, and independent opinions about -/// whether the legacy environment may be deleted: both would report the same write as -/// new and each would keep serving keys the other had deleted. A node that cannot create -/// the lock file has no way to know it is alone, and this is the one migration where -/// being wrong about that destroys data. -/// -/// The lock is an [`Arc`] so the work that relies on it can hold a lease. The startup -/// scan sweeps interrupted writes on the strength of being alone in the directory, and it -/// runs on a thread that outlives the future that started it. -/// -/// # Errors -/// -/// Returns [`Error::Storage`] when another process owns the directory, or when the lock -/// file cannot be created. -fn acquire_store_lock(chunks_dir: &Path) -> Result> { - let path = chunks_dir.join(LOCK_FILE_NAME); - let file = match OpenOptions::new() - .write(true) - .create(true) - .truncate(false) - .open(&path) - { - Ok(f) => f, - // Not a warning and carry on. Without this lock two processes can open the same - // directory, each with its own index, its own view of what is in flight, and its - // own opinion about whether the legacy environment may be deleted. A node that - // cannot take it has no way to know it is alone, and this is the one migration - // where being wrong about that destroys data. - Err(e) => { - return Err(Error::Storage(format!( - "Could not create the chunk store lock {}: {e}. Refusing to start: \ - without it this node cannot tell whether another is using the same data \ - directory. Fix the permissions on that path, or remove a stale lock file \ - left by a different user.", - path.display() - ))) - } - }; - match file.try_lock_exclusive() { - Ok(()) => Ok(Arc::new(file)), - Err(e) => Err(Error::Storage(format!( - "Another process already has the chunk store at {} open ({e}). Two nodes \ - cannot share one data directory: each keeps its own index and they would \ - disagree about what is stored. Stop the other node first.", - chunks_dir.display() - ))), - } -} - -/// What a startup scan found. -struct ScanResult { - /// Every published address, ascending. - keys: Vec, - /// Which shard directories already exist. - shards_present: [bool; SHARD_COUNT], - /// Orphaned temp files removed. - swept_temps: usize, - /// Entries that were neither a chunk nor one of ours. - skipped: usize, -} - -/// Rebuild the key set from directory entries. -/// -/// Reads names only. A `stat` per entry costs about ten times the enumeration on Linux -/// and macOS and fifty to sixty times on Windows, and buys nothing: the filename is the -/// key, and the content is verified on read. -fn scan_store(chunks_dir: &Path) -> Result { - let mut result = ScanResult { - keys: Vec::new(), - shards_present: [false; SHARD_COUNT], - swept_temps: 0, - skipped: 0, - }; - - let top = std::fs::read_dir(chunks_dir).map_err(|e| { - Error::Storage(format!( - "Failed to enumerate chunk store {}: {e}", - chunks_dir.display() - )) - })?; - - for entry in top { - let entry = entry.map_err(|e| { - Error::Storage(format!( - "Failed to read an entry of {}: {e}", - chunks_dir.display() - )) - })?; - let name = entry.file_name(); - let Some(name) = name.to_str() else { - result.skipped = result.skipped.saturating_add(1); - continue; - }; - if name == LAYOUT_FILE_NAME || name == LOCK_FILE_NAME { - continue; - } - if name.starts_with(TEMP_PREFIX) { - if sweep_temp(&entry.path()) { - result.swept_temps = result.swept_temps.saturating_add(1); - } - continue; - } - if name.len() != 2 || !is_lower_hex(name) { - warn!( - "Chunk store: ignoring unexpected entry {name} in {}", - chunks_dir.display() - ); - result.skipped = result.skipped.saturating_add(1); - continue; - } - let Ok(shard) = u8::from_str_radix(name, 16) else { - result.skipped = result.skipped.saturating_add(1); - continue; - }; - // `shards_present` is set inside `scan_shard`, on success only. Setting it from - // the name alone would make a stray regular file called `ab` look like a shard - // that already exists, and every write to that shard would then fail with a - // misleading error until the node was restarted. - scan_shard(&entry.path(), shard, &mut result)?; - } - - result.keys.sort_unstable(); - result.keys.dedup(); - Ok(result) -} - -/// Scan one shard directory into `result`. -fn scan_shard(dir: &Path, shard: u8, result: &mut ScanResult) -> Result<()> { - let entries = match std::fs::read_dir(dir) { - Ok(e) => e, - // A stray file named like a shard, or a directory removed between the two reads. - // Neither is fatal, and neither marks the shard as present. - Err(e) if matches!(e.kind(), ErrorKind::NotFound | ErrorKind::NotADirectory) => { - warn!( - "Chunk store: {} is not a shard directory ({e}); ignoring it", - dir.display() - ); - result.skipped = result.skipped.saturating_add(1); - return Ok(()); - } - // Anything else is a real fault: a permission problem, exhausted descriptors, or - // failing hardware. Opening with a shard's worth of keys silently missing would - // make the node under-claim in its published commitment and stop serving chunks - // it still holds and is answerable for, so refuse to open at all. - Err(e) => { - return Err(Error::Storage(format!( - "Failed to enumerate shard {}: {e}. Refusing to open with an incomplete \ - key set.", - dir.display() - ))) - } - }; - if let Some(slot) = result.shards_present.get_mut(shard as usize) { - *slot = true; - } - - for entry in entries { - let entry = - entry.map_err(|e| Error::Storage(format!("Failed to read {}: {e}", dir.display())))?; - let name = entry.file_name(); - let Some(name) = name.to_str() else { - result.skipped = result.skipped.saturating_add(1); - continue; - }; - if name.starts_with(TEMP_PREFIX) { - if sweep_temp(&entry.path()) { - result.swept_temps = result.swept_temps.saturating_add(1); - } - continue; - } - let Some(key) = decode_chunk_name(name) else { - if name.len() == CHUNK_NAME_LEN && is_hex_any_case(name) { - // A case-folded twin of a real chunk name. On NTFS and default APFS the - // existence check in the write path folds onto it, so a paid write would - // be answered "already stored" and its bytes dropped. Move it aside. - quarantine_entry(&entry.path()); - } else { - warn!( - "Chunk store: ignoring non-chunk entry {name} in {}", - dir.display() - ); - } - result.skipped = result.skipped.saturating_add(1); - continue; - }; - // `file_type` comes from the directory entry itself on Linux and macOS and from - // the enumeration on Windows, so this is not the per-entry `stat` the scan - // deliberately avoids. A pipe, socket, device or directory wearing a chunk name - // must never enter the index: nothing downstream can read it, and it would sit in - // the published commitment forever. - match entry.file_type() { - Ok(kind) if kind.is_file() => {} - Ok(_) => { - warn!( - "Chunk store: {name} in {} is not a regular file; ignoring it", - dir.display() - ); - result.skipped = result.skipped.saturating_add(1); - continue; - } - // Not the same as knowing it is not a file. Treating an unanswered question - // as a no would drop a real chunk from the index and from the commitment - // while its bytes sit on disk, and the node would not serve it again until - // some later restart happened to succeed. Fail the scan instead: an index - // that is missing keys must never be published as this node's key set. - Err(e) => { - return Err(Error::Storage(format!( - "Could not tell what {name} in {} is: {e}. Refusing to publish an \ - index that may be missing chunks.", - dir.display() - ))); - } - } - // A file in the wrong shard is unreachable through `chunk_path`, so indexing it - // would make the index claim a key the read path cannot find. - if shard_index(&key) != shard as usize { - warn!( - "Chunk store: {name} is filed under shard {shard:02x} but belongs in {:02x}; \ - ignoring it. Move it or delete it.", - shard_index(&key) - ); - result.skipped = result.skipped.saturating_add(1); - continue; - } - result.keys.push(key); - } - Ok(()) -} - -/// Remove one orphaned temp file. Returns whether it went. -/// -/// Always removed. The scan that calls this runs only after the store lock has been taken, -/// so by then any temp file is an interrupted write of a previous run and there is no other -/// process that could be writing it. This used to describe a second, gentler mode for the -/// unlocked case; there was never any such branch and there is no caller that would need -/// one. -fn sweep_temp(path: &Path) -> bool { - match std::fs::remove_file(path) { - Ok(()) => { - debug!("Removed orphaned temporary file {}", path.display()); - true - } - Err(e) => { - debug!("Could not remove {}: {e}", path.display()); - false - } - } -} - -/// Open a chunk file, refusing anything that is not a regular file. -/// -/// `Ok(None)` means the file is not there. A named pipe wearing a valid chunk name would -/// otherwise block the opening thread forever: `open` on a FIFO with no writer does not -/// return, and enough of them would exhaust the blocking pool and stall every file and -/// database operation in the process. `O_NOFOLLOW` refuses a symlink for the same reason, -/// and both are checked on the handle rather than the path, so nothing can be swapped -/// underneath between the check and the open. -fn open_regular(path: &Path) -> Result> { - #[cfg(unix)] - let opened = { - use std::os::unix::fs::OpenOptionsExt; - OpenOptions::new() - .read(true) - .custom_flags(libc::O_NONBLOCK | libc::O_NOFOLLOW) - .open(path) - }; - #[cfg(not(unix))] - let opened = OpenOptions::new().read(true).open(path); - - let file = match opened { - Ok(f) => f, - Err(e) if e.kind() == ErrorKind::NotFound => return Ok(None), - Err(e) => { - return Err(Error::Storage(format!( - "Failed to open chunk file {}: {e}", - path.display() - ))) - } - }; - let is_regular = file.metadata().is_ok_and(|m| m.file_type().is_file()); - if !is_regular { - return Err(Error::Storage(format!( - "{} is not a regular file; refusing to read it as a chunk", - path.display() - ))); - } - Ok(Some(file)) -} - -/// Read a chunk file, refusing anything larger than a chunk can legitimately be. -/// -/// A corrupt, sparse, or locally planted file wearing a valid 64-hex name would -/// otherwise be read straight into memory, so a single bad entry could exhaust the node -/// during an ordinary GET or an audit response. -fn read_bounded(file: File, path: &Path) -> Result> { - let ceiling = MAX_CHUNK_SIZE as u64; - let mut buf = Vec::new(); - let read = file.take(ceiling + 1).read_to_end(&mut buf).map_err(|e| { - Error::Storage(format!("Failed to read chunk file {}: {e}", path.display())) - })?; - if read as u64 > ceiling { - return Err(Error::Storage(format!( - "Chunk file {} is larger than the {ceiling} byte maximum; refusing to read it", - path.display() - ))); - } - Ok(buf) -} - -/// Whether a Windows error is one a scanner or indexer holding a handle would produce. -/// -/// `ERROR_ACCESS_DENIED`, `ERROR_SHARING_VIOLATION`, `ERROR_LOCK_VIOLATION`. Every other -/// failure is deterministic and retrying it only burns a blocking thread. -fn is_windows_sharing_violation(e: &std::io::Error) -> bool { - matches!(e.raw_os_error(), Some(5 | 32 | 33)) -} - -/// Publish `temp_path` as `final_path`, retrying a transient sharing violation. -/// -/// On Windows an antivirus scanner or the search indexer can hold a handle to either -/// file for a few milliseconds after it is created, and `MoveFileEx` fails outright -/// rather than queueing. Retrying a bounded number of times turns that from a failed -/// write into a short pause. Every other error returns immediately. -pub(crate) fn rename_with_retry(temp_path: &Path, final_path: &Path) -> std::io::Result<()> { - let mut last = match std::fs::rename(temp_path, final_path) { - Ok(()) => return Ok(()), - Err(e) => e, - }; - if !cfg!(windows) || !is_windows_sharing_violation(&last) { - return Err(last); - } - for attempt in 1..=RENAME_RETRY_ATTEMPTS { - std::thread::sleep(RENAME_RETRY_BACKOFF * attempt); - match std::fs::rename(temp_path, final_path) { - Ok(()) => return Ok(()), - Err(e) => last = e, - } - } - Err(last) -} - -/// Write `payload` and publish it as `final_path`, replacing whatever is there. -/// -/// Success here means the bytes are durable, not merely written. The repair path this -/// serves runs during the pre-retirement pass, where a chunk that fails to match its -/// address is rewritten from the legacy store and the legacy store is then deleted. A -/// replacement that a power loss can undo would leave that chunk with the wrong bytes and -/// no other copy. -fn write_and_replace( - temp_path: &Path, - final_path: &Path, - payload: &[u8], - shard: &Path, -) -> Result<()> { - // Unix: an intra-directory rename is atomic, so a reader sees the old content or the - // new one and never an absence, and the directory flush is what makes it durable. - #[cfg(unix)] - { - write_temp(temp_path, payload)?; - if let Err(e) = rename_with_retry(temp_path, final_path) { - let _ = std::fs::remove_file(temp_path); - return Err(Error::Storage(format!( - "Failed to replace chunk {}: {e}", - final_path.display() - ))); - } - fsync_dir(shard).map_err(|e| { - Error::Storage(format!( - "Replaced {} but could not flush {}: {e}. Not reporting the repair as \ - done, because a rewrite that is not durable must not authorise deleting \ - the copy it was rewritten from.", - final_path.display(), - shard.display() - )) - })?; - Ok(()) - } - // Everywhere else, Windows included: there is no way to flush a directory through the - // standard library, so a rename cannot be shown to be durable at return. Overwriting - // the existing file changes no directory entry at all, and `sync_all` (FlushFileBuffers - // on Windows) is documented to flush the file's data, so a successful return is - // durable under a documented contract. - // - // The cost is that this is not atomic: a crash part-way leaves the file holding a mix - // of old and new bytes. That is safe here and only here, because the only caller that - // matters runs before the legacy store is deleted, and a crash means no report was - // produced and nothing was deleted. The next start re-reads the file, sees it does not - // match its address, and repairs it again from the store that is still there. - #[cfg(not(unix))] - { - let _ = temp_path; - let _ = shard; - let mut file = OpenOptions::new() - .write(true) - .truncate(true) - .open(final_path) - .map_err(|e| { - Error::Storage(format!( - "Failed to open {} for replacement: {e}", - final_path.display() - )) - })?; - file.write_all(payload).map_err(|e| { - Error::Storage(format!("Failed to rewrite {}: {e}", final_path.display())) - })?; - file.sync_all().map_err(|e| { - Error::Storage(format!( - "Rewrote {} but could not flush it: {e}. Not reporting the repair as \ - done, because a rewrite that is not durable must not authorise deleting \ - the copy it was rewritten from.", - final_path.display() - )) - })?; - Ok(()) - } -} - -/// Create `temp_path`, write `payload` into it, and flush it. -/// -/// Flushed before any rename. On ext4 `auto_da_alloc` only orders the data before the -/// rename's own commit; it does not make the data durable, and btrfs has been observed -/// reordering. A name must never become visible on bytes that are not on the platter. -fn write_temp(temp_path: &Path, payload: &[u8]) -> Result<()> { - let mut f = OpenOptions::new() - .write(true) - .create_new(true) - .open(temp_path) - .map_err(|e| { - Error::Storage(format!( - "Failed to create temporary file {}: {e}", - temp_path.display() - )) - })?; - if let Err(e) = f.write_all(payload) { - let _ = std::fs::remove_file(temp_path); - return Err(Error::Storage(format!( - "Failed to write {}: {e}", - temp_path.display() - ))); - } - if let Err(e) = f.sync_all() { - let _ = std::fs::remove_file(temp_path); - return Err(Error::Storage(format!( - "Failed to flush {}: {e}", - temp_path.display() - ))); - } - Ok(()) -} - -/// Write `payload` and publish it under `final_path`. -/// -/// The temp lives in the destination directory, so the publish is an intra-directory -/// rename: atomic on every filesystem we support, and needing only that one directory -/// Put `payload` on disk as `final_path`, durably. -/// -/// Returns [`PutOutcome::Duplicate`] when the name is already taken. The name is a hash -/// of the content, so that is not treated as proof the bytes are right: the caller -/// re-reads and verifies them. -#[cfg(unix)] -fn publish( - temp_path: &Path, - final_path: &Path, - payload: &[u8], - shard: &Path, -) -> std::result::Result { - // On Unix nothing is ever created under the final name by a failing path: the bytes go - // to a temporary and only a successful rename gives them the real name. So every - // failure here leaves the name as it found it. - publish_via_rename(temp_path, final_path, payload, shard) - .map_err(PublishFailed::nothing_written) -} - -/// Put `payload` on disk as `final_path`, durably. See [`publish_in_place`] for why this -/// takes a different route off Unix. -#[cfg(not(unix))] -fn publish( - temp_path: &Path, - final_path: &Path, - payload: &[u8], - shard: &Path, -) -> std::result::Result { - let _ = temp_path; - let _ = shard; - publish_in_place(final_path, payload) -} - -/// Create the chunk under its final name and flush it. Everywhere but Unix. -/// -/// There is no way to flush a directory through the standard library, and Microsoft does -/// not document `MoveFileEx` as durable at return unless it is called with -/// `MOVEFILE_WRITE_THROUGH`, which std does not use. So off Unix a rename cannot be -/// relied on to have reached the disk before the legacy store is deleted. -/// -/// Creating the file under its final name sidesteps the rename entirely. Microsoft -/// documents that creation metadata is cached and that `FlushFileBuffers`, which -/// `sync_all` calls on Windows, is the way to flush it. So a successful create, write and -/// flush is a durable publication under a documented contract, with no directory flush -/// and no rename involved. -/// -/// The cost is that a crash mid-write leaves a partial file wearing a real chunk name. -/// That is why a duplicate re-reads and verifies rather than trusting the name, and why -/// the pre-retirement pass re-hashes everything before anything is deleted. -#[cfg(not(unix))] -fn publish_in_place( - final_path: &Path, - payload: &[u8], -) -> std::result::Result { - // Test-only, and here rather than after the write so that it means the same thing on - // both platforms: the file half of a dual write has not happened yet. On Unix the - // equivalent point is the temporary file written and the rename not yet made, which is - // also before the chunk's name exists on disk. Stopping after the write instead would - // put the file under its real name already, so a crash there is not between the two - // halves at all, and it could not demonstrate anything about the missing flush either: - // killing a process does not empty the page cache, so the bytes are still there to be - // read. Only losing power loses them, which no test that kills a process can stage. - #[cfg(any(test, feature = "test-utils"))] - halt_here_if_asked(HALT_BEFORE_PUBLISH, final_path); - let mut file = match OpenOptions::new() - .write(true) - .create_new(true) - .open(final_path) - { - Ok(f) => f, - // Someone got there first. Immutable content under a content-addressed name, so - // the caller verifies what is already there rather than assuming it is right. - Err(e) if e.kind() == ErrorKind::AlreadyExists => return Ok(PutOutcome::Duplicate), - Err(e) => { - // Nothing was created, so nothing was spent. - return Err(PublishFailed::nothing_written(Error::Storage(format!( - "Failed to create chunk {}: {e}", - final_path.display() - )))); - } - }; - if let Err(e) = file.write_all(payload) { - drop(file); - // Taken back if it can be. Whether it could is what the caller needs: the file - // was created by this call, so if it is still there the space is spent. - let left_behind = std::fs::remove_file(final_path).is_err(); - return Err(PublishFailed { - error: Error::Storage(format!("Failed to write {}: {e}", final_path.display())), - left_behind, - }); - } - if let Err(e) = file.sync_all() { - drop(file); - // Taken back if it can be. Whether it could is what the caller needs: the file - // was created by this call, so if it is still there the space is spent. - let left_behind = std::fs::remove_file(final_path).is_err(); - return Err(PublishFailed { - error: Error::Storage(format!("Failed to flush {}: {e}", final_path.display())), - left_behind, - }); - } - Ok(PutOutcome::New) -} - -/// Write a temp beside the target and rename it into place. Unix only. -/// -/// Places the bytes and nothing more. Making the name durable is -/// [`flush_publication`]'s job, kept separate so a caller can tell a publish that spent no -/// space from one that spent it and could not be reported. -#[cfg(unix)] -fn publish_via_rename( - temp_path: &Path, - final_path: &Path, - payload: &[u8], - _shard: &Path, -) -> Result { - // Content is immutable and the name is its hash, so an existing file already holds - // exactly these bytes. Skipping the write is both cheaper and safer than replacing - // it: on Windows a rename over a file another thread has open fails outright. - // - // The caller flushes either way. A name that is already there is not proof it is - // durable: - // the write that put it there may have been this store's own previous attempt, whose - // rename landed and whose directory flush then failed. That attempt returned an - // error, so nothing was retired on the strength of it, but if this call reported a - // durable duplicate without flushing, the retry would silently launder an unflushed - // rename into a copy that authorises deleting the last other one. - let outcome = if final_path.exists() { - PutOutcome::Duplicate - } else { - write_temp(temp_path, payload)?; - // Test-only: the one moment a complete chunk exists on disk under a name nothing - // looks for. A crash test needs to die at a named point rather than wherever a - // sleep in another process happened to land. - #[cfg(any(test, feature = "test-utils"))] - halt_here_if_asked(HALT_BEFORE_PUBLISH, temp_path); - match rename_with_retry(temp_path, final_path) { - Ok(()) => PutOutcome::New, - Err(e) => { - let _ = std::fs::remove_file(temp_path); - // Another writer of the same address won the race, or the destination was - // open. Either way the bytes are already published. - if !final_path.exists() { - return Err(Error::Storage(format!( - "Failed to publish chunk {}: {e}", - final_path.display() - ))); - } - PutOutcome::Duplicate - } - } - }; - - Ok(outcome) -} - -/// A publish that failed, and whether it left its bytes on the disk. -/// -/// The second half is the point. A failure before anything was created has spent nothing; -/// one that created the file and then could not remove it again has spent the space, and -/// whoever is accounting for free space has to know which happened. Only the code that did -/// the creating can say. -struct PublishFailed { - error: Error, - left_behind: bool, -} - -impl PublishFailed { - /// A failure that created nothing. - fn nothing_written(error: Error) -> Self { - Self { - error, - left_behind: false, - } - } -} - -/// Make a publication durable by flushing the directory its name lives in. -/// -/// Separate from placing the bytes, because the caller has to tell the two failures apart. -/// A publish that fails before the bytes land has spent nothing; one that fails here has -/// spent the space and must not be reported as stored, so whoever is accounting for free -/// space has to charge it while whoever is accounting for chunks must not count it. -/// -/// NOT best effort. The directory flush is what makes the rename durable, and a copy -/// reported successful is what authorises deleting the only other copy. Swallowing the -/// failure would let a power loss discard the directory entry after the legacy store had -/// already been removed. -/// -/// # Errors -/// -/// Returns [`Error::Storage`] if the directory cannot be flushed. -#[cfg(unix)] -fn flush_publication(final_path: &Path, shard: &Path) -> Result<()> { - fsync_dir(shard).map_err(|e| { - Error::Storage(format!( - "Published {} but could not flush {}: {e}. Not reporting this chunk as stored, \ - because a copy that is not durable must not authorise deleting another.", - final_path.display(), - shard.display() - )) - }) -} - -/// Nothing to do off Unix, where the chunk is created under its final name and flushed -/// with `sync_all`, which is documented to carry its creation metadata with it, and where -/// there is no way to flush a directory at all. -#[cfg(not(unix))] -fn flush_publication(_final_path: &Path, _shard: &Path) -> Result<()> { - Ok(()) -} - -#[cfg(test)] -#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] -mod tests { - use super::*; - use std::collections::HashSet; - - /// A directory flush that fails must say so. - /// - /// The quiet version of this function is only used where the answer does not change - /// what happens next. On the publish path it does. - #[cfg(unix)] - #[test] - fn flushing_a_directory_that_is_not_there_reports_the_failure() { - let dir = TempDir::new().expect("temp dir"); - assert!(fsync_dir(dir.path()).is_ok()); - assert!(fsync_dir(&dir.path().join("no-such-shard")).is_err()); - } - - /// A chunk whose directory entry was never flushed is not reported as stored. - /// - /// This is the whole safety argument for retirement: the legacy store is deleted - /// because every chunk was copied durably. A published file whose directory flush - /// failed can vanish on power loss, so counting it as copied would lose data. The - /// file staying on disk afterwards is fine, the next pass republishes it. - #[cfg(unix)] - #[test] - fn a_publish_whose_directory_flush_fails_is_not_reported_as_stored() { - let dir = TempDir::new().expect("temp dir"); - let temp_path = dir.path().join("chunk.tmp"); - let final_path = dir.path().join("chunk"); - let unflushable = dir.path().join("shard-that-does-not-exist"); - - // Asserted in two steps, not chained. Chaining them means a regression in placing - // the bytes also produces an error, and the test passes without the flush ever - // being reached: it would be checking that something went wrong rather than that - // this went wrong. - let placed = publish_via_rename(&temp_path, &final_path, b"payload", &unflushable); - assert!( - placed.is_ok(), - "the bytes must be placed before this can be about the flush: {:?}", - placed.err() - ); - let outcome = flush_publication(&final_path, &unflushable); - - assert!( - outcome.is_err(), - "an unflushed publication must not be reported as stored" - ); - assert!( - !temp_path.exists(), - "the temp file must not be left behind either way" - ); - } - - use tempfile::TempDir; - - /// Open a store on a fresh temp directory with the disk reserve disabled. - async fn test_store() -> (FileStore, TempDir) { - let dir = TempDir::new().expect("temp dir"); - let store = FileStore::new(FileStoreConfig { - root_dir: dir.path().to_path_buf(), - verify_on_read: true, - disk_reserve: 0, - }) - .await - .expect("open store"); - (store, dir) - } - - /// Open a store on an existing directory, as a restart would. - async fn reopen(dir: &TempDir) -> FileStore { - FileStore::new(FileStoreConfig { - root_dir: dir.path().to_path_buf(), - verify_on_read: true, - disk_reserve: 0, - }) - .await - .expect("reopen store") - } - - /// An ordinary read settles whether the node answers for a chunk. - /// - /// Not only the reads that were checking something. A read that failed means the - /// chunk cannot be served, whoever asked; a read that worked means it can be. Deciding - /// this anywhere else leaves a key stuck unadvertised after the fault has cleared, or - /// advertised after it has not. - #[cfg(unix)] - #[tokio::test] - async fn an_ordinary_read_decides_whether_the_node_answers_for_a_chunk() { - use std::os::unix::fs::PermissionsExt; - - let (store, dir) = test_store().await; - let (addr, content) = addressed("read-decides"); - store.put(&addr, &content).await.expect("put"); - let path = store.chunk_path(&addr); - - let mut perms = std::fs::metadata(&path).expect("meta").permissions(); - perms.set_mode(0o000); - std::fs::set_permissions(&path, perms).expect("chmod"); - - assert!(store.get(&addr).await.is_err(), "the read must fail"); - assert!( - !store.exists(&addr).expect("exists"), - "and a plain read that failed must stop the node answering for it" - ); - - let mut perms = std::fs::metadata(&path).expect("meta").permissions(); - perms.set_mode(0o600); - std::fs::set_permissions(&path, perms).expect("chmod back"); - - assert_eq!( - store.get(&addr).await.expect("get").expect("present"), - content - ); - assert!( - store.exists(&addr).expect("exists"), - "and a plain read that worked must start it answering again" - ); - drop(dir); - } - - /// Two writes for one key: waiting means waiting for both. - /// - /// Cancellation releases the caller's lane while the blocking half survives, so a - /// second write for the same key can start behind the first. If the registry only - /// recorded that *something* was writing, whichever finished first would clear it and - /// a delete would be told the key was free while the other was still queued, then be - /// undone by it. - #[tokio::test] - async fn waiting_for_a_key_waits_for_every_write_of_it() { - let (store, dir) = test_store().await; - let store = Arc::new(store); - let (addr, content) = addressed("two-writers"); - - // Two registrations, as two overlapping writes would make. - let first = store.begin_write(&addr); - let second = store.begin_write(&addr); - - let waiting = { - let store = Arc::clone(&store); - tokio::spawn(async move { store.wait_for_write(&addr).await }) - }; - tokio::time::sleep(Duration::from_millis(50)).await; - assert!(!waiting.is_finished()); - - // One finishes. The other has not, so the wait must continue. - drop(first); - tokio::time::sleep(Duration::from_millis(50)).await; - assert!( - !waiting.is_finished(), - "one write finishing does not mean the key is free" - ); - - drop(second); - waiting - .await - .expect("the wait ends once both have finished"); - - // And the store is still usable afterwards. - store.put(&addr, &content).await.expect("put"); - assert!(store.exists(&addr).expect("exists")); - drop(dir); - } - - /// A chunk this store cannot read is kept but not claimed. - /// - /// Both halves matter. Deleting it, or dropping it from the index, is how a chunk ends - /// up in neither this store's view nor the legacy one, which is what retirement - /// destroys. Claiming it anyway puts the key in signed commitments and answers - /// presence probes with a yes for a chunk the node cannot serve, and the audit that - /// catches that still penalises. - #[cfg(unix)] - #[tokio::test] - async fn a_chunk_that_cannot_be_read_is_kept_but_not_claimed() { - use std::os::unix::fs::PermissionsExt; - - let (store, dir) = test_store().await; - let (addr, content) = addressed("unreadable-for-now"); - store.put(&addr, &content).await.expect("put"); - assert!(store.exists(&addr).expect("exists")); - - let path = store.chunk_path(&addr); - let mut perms = std::fs::metadata(&path).expect("meta").permissions(); - perms.set_mode(0o000); - std::fs::set_permissions(&path, perms).expect("chmod"); - - // Offering the same bytes again must not be acknowledged, and must not replace - // what is there on the strength of a read that did not happen. - assert!( - store.put(&addr, &content).await.is_err(), - "an unreadable chunk must not be reported as stored" - ); - assert!(path.exists(), "and the file must be left alone"); - assert!( - !store.exists(&addr).expect("exists"), - "but the node must stop claiming it" - ); - assert!(!store.all_keys().await.expect("keys").contains(&addr)); - - // Readable again: the node answers for it once more. - let mut perms = std::fs::metadata(&path).expect("meta").permissions(); - perms.set_mode(0o600); - std::fs::set_permissions(&path, perms).expect("chmod back"); - assert!(!store.put(&addr, &content).await.expect("put again")); - assert!(store.exists(&addr).expect("exists")); - assert!(store.all_keys().await.expect("keys").contains(&addr)); - drop(dir); - } - - /// Content plus the address it hashes to. - fn addressed(seed: &str) -> (XorName, Vec) { - let content = format!("chunk-content-{seed}").into_bytes(); - (crate::client::compute_address(&content), content) - } - - #[tokio::test] - async fn put_then_get_returns_the_same_bytes() { - let (store, _dir) = test_store().await; - let (addr, content) = addressed("a"); - - assert!(store.put(&addr, &content).await.expect("put")); - let got = store.get(&addr).await.expect("get").expect("present"); - assert_eq!(got, content); - } - - #[tokio::test] - async fn a_second_put_of_the_same_chunk_reports_not_new() { - let (store, _dir) = test_store().await; - let (addr, content) = addressed("b"); - - assert!(store.put(&addr, &content).await.expect("first put")); - assert!(!store.put(&addr, &content).await.expect("second put")); - assert_eq!(store.current_chunks().expect("count"), 1); - assert_eq!(store.stats().duplicates, 1); - } - - #[tokio::test] - async fn get_of_an_unknown_address_is_none() { - let (store, _dir) = test_store().await; - let (addr, _) = addressed("missing"); - assert!(store.get(&addr).await.expect("get").is_none()); - } - - #[tokio::test] - async fn exists_tracks_the_store() { - let (store, _dir) = test_store().await; - let (addr, content) = addressed("c"); - - assert!(!store.exists(&addr).expect("exists")); - store.put(&addr, &content).await.expect("put"); - assert!(store.exists(&addr).expect("exists")); - store.delete(&addr).await.expect("delete"); - assert!(!store.exists(&addr).expect("exists")); - } - - #[tokio::test] - async fn delete_unlinks_the_file_and_returns_the_space() { - let (store, _dir) = test_store().await; - let (addr, content) = addressed("d"); - store.put(&addr, &content).await.expect("put"); - - let path = store.chunk_path(&addr); - assert!(path.exists(), "the chunk file should be on disk"); - - assert!(store.delete(&addr).await.expect("delete")); - assert!(!path.exists(), "delete must actually unlink the file"); - assert_eq!(store.current_chunks().expect("count"), 0); - - // Deleting again is a no-op that reports nothing was there. - assert!(!store.delete(&addr).await.expect("second delete")); - } - - #[tokio::test] - async fn content_that_does_not_hash_to_its_address_is_rejected() { - let (store, _dir) = test_store().await; - let (addr, _) = addressed("e"); - let err = store - .put(&addr, b"different content") - .await - .expect_err("must reject"); - assert!( - format!("{err}").contains("Content address mismatch"), - "unexpected error: {err}" - ); - assert_eq!(store.current_chunks().expect("count"), 0); - } - - #[tokio::test] - async fn a_chunk_is_filed_under_the_last_two_hex_characters_of_its_address() { - let (store, dir) = test_store().await; - let (addr, content) = addressed("f"); - store.put(&addr, &content).await.expect("put"); - - let name = hex::encode(addr); - let expected_shard = name - .get(name.len() - 2..) - .expect("64-character name") - .to_string(); - let path = dir - .path() - .join(CHUNKS_DIR_NAME) - .join(&expected_shard) - .join(&name); - assert!(path.exists(), "expected the chunk at {}", path.display()); - } - - #[tokio::test] - async fn the_index_is_rebuilt_from_the_filesystem_on_restart() { - let (store, dir) = test_store().await; - let mut written = Vec::new(); - for i in 0..64 { - let (addr, content) = addressed(&format!("restart-{i}")); - store.put(&addr, &content).await.expect("put"); - written.push(addr); - } - drop(store); - - let reopened = reopen(&dir).await; - assert_eq!(reopened.current_chunks().expect("count"), 64); - for addr in &written { - assert!(reopened.exists(addr).expect("exists"), "lost a key"); - } - } - - #[tokio::test] - async fn all_keys_is_sorted_ascending() { - let (store, dir) = test_store().await; - for i in 0..128 { - let (addr, content) = addressed(&format!("sorted-{i}")); - store.put(&addr, &content).await.expect("put"); - } - - let keys = store.all_keys().await.expect("all_keys"); - let mut sorted = keys.clone(); - sorted.sort_unstable(); - assert_eq!(keys, sorted, "all_keys() must be ordered"); - - // And the order has to survive a restart, because the commitment builder - // truncates the responsible subset before the Merkle tree sorts it. - drop(store); - let reopened = reopen(&dir).await; - assert_eq!(reopened.all_keys().await.expect("all_keys"), keys); - } - - #[tokio::test] - async fn get_raw_skips_verification() { - let (store, _dir) = test_store().await; - let (addr, content) = addressed("raw"); - store.put(&addr, &content).await.expect("put"); - - // Corrupt the file behind the store's back. - std::fs::write(store.chunk_path(&addr), b"tampered").expect("tamper"); - - let raw = store.get_raw(&addr).await.expect("get_raw").expect("bytes"); - assert_eq!(raw, b"tampered"); - } - - /// A put whose caller goes away does not admit a key on bytes nothing has read. - /// - /// The blocking half of a put outlives the future that started it, deliberately, so - /// the work is never left half done. That makes anything it writes to memory a claim - /// the node keeps whether or not the caller is still there to finish checking it. - /// - /// For a chunk this call published the claim is earned: the bytes were hashed against - /// their own name on the way in. For a name that was already taken it is not. The - /// check that decides whether those bytes are good runs after the await, and a dropped - /// future skips it, so admitting the key in the closure claims a chunk nobody read. - /// - /// Staged with a fifo, which is the sharpest case and a real one: the startup scan - /// refuses non-regular entries by design, so this is a key the store has already - /// decided it must not claim, walked in through the back door. - #[cfg(unix)] - #[tokio::test] - // The gate is held across an await deliberately: holding it is what parks the put - // inside its closure, which is the state under test. Dropping it before awaiting would - // let the put finish and there would be nothing to cancel. - #[allow(clippy::await_holding_lock)] - async fn a_cancelled_put_does_not_admit_a_key_whose_bytes_were_never_read() { - let dir = TempDir::new().expect("temp dir"); - let store = Arc::new(reopen(&dir).await); - - // A name a real chunk would use, wearing something that is not a chunk. - let content = b"the bytes that belong under this name".to_vec(); - let addr = crate::client::compute_address(&content); - let shard = dir - .path() - .join(CHUNKS_DIR_NAME) - .join(format!("{:02x}", addr[31])); - std::fs::create_dir_all(&shard).expect("mkdir"); - let path = shard.join(hex::encode(addr)); - let name = std::ffi::CString::new(path.as_os_str().as_encoded_bytes()) - .expect("a path with no interior nul"); - // SAFETY: `name` is a valid NUL-terminated C string that outlives the call, and the - // mode is a constant. `mkfifo` reads the pointer and returns; nothing is retained. - #[allow(clippy::undocumented_unsafe_blocks, unsafe_code)] - let made = unsafe { libc::mkfifo(name.as_ptr(), 0o644) }; - assert_eq!(made, 0, "could not make the fifo this test needs"); - - // Hold the gate so the put parks inside the closure, then drop the future while it - // is parked. That is a caller going away mid-put, which is what a cancelled - // request, a client disconnect or a shutdown all look like from in here. - let gate = store.test_put_gate(); - let held = gate.write(); - let put = { - let store = Arc::clone(&store); - let content = content.clone(); - tokio::spawn(async move { store.put(&addr, &content).await }) - }; - // Waited for rather than slept at. A sleep proves nothing: if the put had not - // reached the gated closure yet, aborting would cancel it before it ever got - // there and the test would pass having staged nothing. - let deadline = std::time::Instant::now() + Duration::from_secs(30); - while store.tasks_in_flight() == 0 { - assert!( - std::time::Instant::now() < deadline, - "the put never reached the closure, so there was nothing to cancel" - ); - tokio::time::sleep(Duration::from_millis(5)).await; - } - put.abort(); - let _ = put.await; - drop(held); - store.wait_idle().await; - - assert!( - !store.is_indexed(&addr), - "a cancelled put admitted {} on bytes nothing read; the fifo under that name \ - would then be advertised, committed to, and audited against", - hex::encode(addr) - ); - assert!( - !store.exists(&addr).unwrap_or(true), - "and the node must not claim it either" - ); - } - - /// A marker temporary left in the node root is swept, and nothing else is. - /// - /// The migration marker is written next to itself in the root, which no sweep looked - /// at, so a crash between its write and its rename left one there for the life of the - /// node. Small, but nothing was ever going to remove it. - /// - /// The second half is the point: this runs over a directory holding a node's data, so - /// it has to take only the exact shape this module writes and leave everything else - /// where it is. - #[tokio::test] - async fn a_leftover_marker_temporary_is_swept_and_its_neighbours_are_not() { - let dir = TempDir::new().expect("temp dir"); - let root = dir.path(); - let leftover = root.join(format!("{TEMP_PREFIX}1234.abcdef01.marker")); - std::fs::write(&leftover, b"an interrupted marker write").expect("plant"); - - // Things that must survive: the marker itself, a chunk-shaped temp that belongs to - // the chunk tree's own sweep, and anything an operator put there. - let keep = [ - root.join("migration-state.json"), - root.join(format!("{TEMP_PREFIX}1234.abcdef01.chunk")), - root.join("notes.txt"), - // Prefix and suffix alone would take these. The pid and the nonce are checked - // because this runs over a directory holding a node's data. - root.join(format!("{TEMP_PREFIX}operator-notes.marker")), - root.join(format!("{TEMP_PREFIX}1234.nothex01.marker")), - root.join(format!("{TEMP_PREFIX}1234.abcdef0.marker")), - root.join(format!("{TEMP_PREFIX}1234.abcdef01.extra.marker")), - ]; - for path in &keep { - std::fs::write(path, b"keep me").expect("plant"); - } - - let store = reopen(&dir).await; - drop(store); - - assert!( - !leftover.exists(), - "the leftover marker temporary is still in the node root" - ); - for path in &keep { - assert!( - path.exists(), - "{} was swept and should not have been", - path.display() - ); - } - } - - #[tokio::test] - async fn a_corrupt_chunk_is_removed_so_replication_can_repair_it() { - let (store, _dir) = test_store().await; - let (addr, content) = addressed("corrupt"); - store.put(&addr, &content).await.expect("put"); - std::fs::write(store.chunk_path(&addr), b"tampered").expect("tamper"); - - let err = store.get(&addr).await.expect_err("verification must fail"); - assert!(format!("{err}").contains("verification failed"), "{err}"); - - assert!(!store.chunk_path(&addr).exists(), "corrupt file must go"); - assert!(!store.exists(&addr).expect("exists")); - assert!( - !store.all_keys().await.expect("all_keys").contains(&addr), - "a corrupt chunk must stop being advertised" - ); - assert_eq!(store.stats().verification_failures, 1); - } - - #[tokio::test] - async fn a_file_removed_underneath_the_store_drops_out_of_the_index() { - let (store, _dir) = test_store().await; - let (addr, content) = addressed("vanished"); - store.put(&addr, &content).await.expect("put"); - - std::fs::remove_file(store.chunk_path(&addr)).expect("remove behind our back"); - - assert!(store.get(&addr).await.expect("get").is_none()); - assert!(!store.exists(&addr).expect("exists")); - assert_eq!(store.current_chunks().expect("count"), 0); - } - - #[tokio::test] - async fn interrupted_writes_are_swept_at_startup() { - let (store, dir) = test_store().await; - let (addr, content) = addressed("sweep"); - store.put(&addr, &content).await.expect("put"); - let shard = store - .chunk_path(&addr) - .parent() - .expect("shard") - .to_path_buf(); - drop(store); - - let orphan = shard.join(format!("{TEMP_PREFIX}999.7")); - std::fs::write(&orphan, b"half a chunk").expect("write orphan"); - let stray_root = dir - .path() - .join(CHUNKS_DIR_NAME) - .join(format!("{TEMP_PREFIX}999.8")); - std::fs::write(&stray_root, b"half a marker").expect("write stray"); - - let reopened = reopen(&dir).await; - assert!(!orphan.exists(), "an interrupted write must not survive"); - assert!(!stray_root.exists(), "nor one at the store root"); - assert_eq!(reopened.current_chunks().expect("count"), 1); - } - - #[tokio::test] - async fn concurrent_writers_of_one_address_store_it_exactly_once() { - let (store, _dir) = test_store().await; - let store = Arc::new(store); - let (addr, content) = addressed("racing"); - - let mut tasks = Vec::new(); - for _ in 0..16 { - let store = Arc::clone(&store); - let content = content.clone(); - tasks.push(tokio::spawn( - async move { store.put(&addr, &content).await }, - )); - } - - let mut new_count = 0; - for task in tasks { - if task.await.expect("join").expect("put") { - new_count += 1; - } - } - assert_eq!(new_count, 1, "exactly one writer may report a new chunk"); - assert_eq!(store.current_chunks().expect("count"), 1); - assert_eq!( - store.get(&addr).await.expect("get").expect("present"), - content - ); - } - - #[tokio::test] - async fn names_that_are_not_lowercase_hex_are_ignored_by_the_scan() { - let (store, dir) = test_store().await; - let (addr, content) = addressed("scan"); - store.put(&addr, &content).await.expect("put"); - let shard = store - .chunk_path(&addr) - .parent() - .expect("shard") - .to_path_buf(); - drop(store); - - // Uppercase is deliberately rejected: on a case-folding filesystem accepting it - // would let one file answer to two index entries. - let upper = shard.join(hex::encode_upper(addressed("upper").0)); - std::fs::write(&upper, b"x").expect("write upper"); - std::fs::write(shard.join("not-a-chunk"), b"x").expect("write junk"); - std::fs::write(shard.join("deadbeef"), b"x").expect("write short"); - - let reopened = reopen(&dir).await; - assert_eq!(reopened.current_chunks().expect("count"), 1); - } - - #[tokio::test] - async fn a_chunk_filed_in_the_wrong_shard_is_not_indexed() { - let (store, dir) = test_store().await; - let (addr, content) = addressed("misfiled"); - store.put(&addr, &content).await.expect("put"); - drop(store); - - // Move it one shard over: the read path would never find it there, so indexing - // it would make the store advertise a key it cannot serve. - let correct = dir - .path() - .join(CHUNKS_DIR_NAME) - .join(shard_name(&addr)) - .join(hex::encode(addr)); - let wrong_shard_index = (shard_index(&addr) + 1) % SHARD_COUNT; - let wrong_dir = dir - .path() - .join(CHUNKS_DIR_NAME) - .join(format!("{wrong_shard_index:02x}")); - std::fs::create_dir_all(&wrong_dir).expect("mkdir"); - std::fs::rename(&correct, wrong_dir.join(hex::encode(addr))).expect("misfile"); - - let reopened = reopen(&dir).await; - assert_eq!(reopened.current_chunks().expect("count"), 0); - assert!(!reopened.exists(&addr).expect("exists")); - } - - #[tokio::test] - async fn the_layout_marker_is_written_once_and_checked_on_reopen() { - let (store, dir) = test_store().await; - drop(store); - - let marker = dir.path().join(CHUNKS_DIR_NAME).join(LAYOUT_FILE_NAME); - let layout: StoreLayout = - serde_json::from_slice(&std::fs::read(&marker).expect("read marker")) - .expect("parse marker"); - assert_eq!(layout, StoreLayout::default()); - - // A store written by a future build must be refused, not misread. - let future = StoreLayout { - schema: LAYOUT_SCHEMA + 1, - ..StoreLayout::default() - }; - std::fs::write(&marker, serde_json::to_vec(&future).expect("encode")).expect("write"); - let err = FileStore::new(FileStoreConfig { - root_dir: dir.path().to_path_buf(), - verify_on_read: true, - disk_reserve: 0, - }) - .await - .expect_err("must refuse a newer layout"); - assert!(format!("{err}").contains("newer than this build"), "{err}"); - } - - #[tokio::test] - async fn an_unknown_shard_scheme_is_refused() { - let (store, dir) = test_store().await; - drop(store); - let marker = dir.path().join(CHUNKS_DIR_NAME).join(LAYOUT_FILE_NAME); - let other = StoreLayout { - scheme: "prefix-hex".to_string(), - ..StoreLayout::default() - }; - std::fs::write(&marker, serde_json::to_vec(&other).expect("encode")).expect("write"); - let err = FileStore::new(FileStoreConfig { - root_dir: dir.path().to_path_buf(), - verify_on_read: true, - disk_reserve: 0, - }) - .await - .expect_err("must refuse an unknown scheme"); - assert!(format!("{err}").contains("shard scheme"), "{err}"); - } - - #[tokio::test] - async fn writes_are_refused_when_the_disk_reserve_cannot_be_met() { - let dir = TempDir::new().expect("temp dir"); - let store = FileStore::new(FileStoreConfig { - root_dir: dir.path().to_path_buf(), - verify_on_read: true, - disk_reserve: u64::MAX / 2, - }) - .await - .expect("open store"); - - let (addr, content) = addressed("full"); - let err = store.put(&addr, &content).await.expect_err("must refuse"); - assert!( - format!("{err}").contains("Insufficient disk space"), - "{err}" - ); - assert!(store.check_capacity().is_err()); - } - - #[tokio::test] - async fn capacity_is_size_aware() { - // Wide enough that a test running alongside this one cannot move the answer. - const MARGIN: u64 = 512 * 1024 * 1024; - - let dir = TempDir::new().expect("temp dir"); - let available = fs2::available_space(dir.path()).expect("free space"); - // A reserve that leaves room for a small write but not a huge one. This is the - // whole reason the predicate takes a size: free bytes alone stopped being a - // sufficient answer once chunks became files. - let store = FileStore::new(FileStoreConfig { - root_dir: dir.path().to_path_buf(), - verify_on_read: true, - disk_reserve: available.saturating_sub(MARGIN), - }) - .await - .expect("open store"); - - assert!(store.check_capacity_for(1024).is_ok()); - assert!(store.check_capacity_for(4 * MARGIN).is_err()); - } - - #[test] - fn suffix_shards_stay_uniform_for_a_close_group_of_keys() { - // The real distribution: a node holds keys it is closest to, so they share a - // long leading prefix with its own ID. Sharding on that prefix collapses to one - // directory. The trailing byte is untouched by close-group membership. - let mut prefix_dirs = HashSet::new(); - let mut suffix_dirs = HashSet::new(); - for i in 0u32..4096 { - let mut key = [0u8; XORNAME_LEN]; - // 20 shared leading bits, as a ~1M-node network would impose. - let tail = crate::client::compute_address(&i.to_le_bytes()); - key.copy_from_slice(&tail); - if let Some(b) = key.first_mut() { - *b = 0xab; - } - if let Some(b) = key.get_mut(1) { - *b = 0xcd; - } - if let Some(b) = key.get_mut(2) { - *b &= 0x0f; - } - prefix_dirs.insert(key.first().copied().unwrap_or(0)); - suffix_dirs.insert(shard_index(&key)); - } - assert_eq!( - prefix_dirs.len(), - 1, - "prefix sharding collapses for a node's own holdings" - ); - assert!( - suffix_dirs.len() > 250, - "suffix sharding must stay uniform, got {} of 256 directories", - suffix_dirs.len() - ); - } - - #[test] - fn no_chunk_filename_can_spell_a_reserved_windows_device_name() { - // Hex has no `n`, `u`, `x`, `p`, `r`, `l`, `t`, `o` or `s`, so `CON`, `NUL`, - // `AUX`, `PRN`, `COM1` and `LPT1` are all unspellable at any length. This is why - // the encoding is hex and not base32 or base64url. - for reserved in ["con", "prn", "aux", "nul", "com1", "com9", "lpt1", "lpt9"] { - assert!( - !is_lower_hex(reserved), - "{reserved} must not be a valid chunk or shard name" - ); - } - } - - #[test] - fn only_full_length_lowercase_hex_decodes_to_an_address() { - // 0xab so the hex form actually contains letters, which is where case matters. - assert!(decode_chunk_name(&hex::encode([0xabu8; XORNAME_LEN])).is_some()); - assert!(decode_chunk_name(&hex::encode_upper([0xabu8; XORNAME_LEN])).is_none()); - assert!(decode_chunk_name("deadbeef").is_none()); - assert!(decode_chunk_name("").is_none()); - assert!(decode_chunk_name(&"g".repeat(CHUNK_NAME_LEN)).is_none()); - } - - #[tokio::test] - async fn repair_replaces_bad_bytes_without_the_file_ever_being_absent() { - let (store, _dir) = test_store().await; - let (addr, content) = addressed("repairable"); - store.put(&addr, &content).await.expect("put"); - let path = store.chunk_path(&addr); - - std::fs::write(&path, b"rotted").expect("corrupt"); - store.repair(&addr, &content).await.expect("repair"); - - assert_eq!( - store.get(&addr).await.expect("get").expect("present"), - content - ); - assert!(store.exists(&addr).expect("exists")); - } - - #[tokio::test] - async fn a_repair_with_the_wrong_bytes_is_refused_and_changes_nothing() { - let (store, _dir) = test_store().await; - let (addr, content) = addressed("guarded"); - store.put(&addr, &content).await.expect("put"); - let path = store.chunk_path(&addr); - - // The whole point of repairing in place is that a failure must leave the old file - // where it was. Deleting first and writing after would open a window whose only - // surviving copy is the one the caller is about to destroy. - let err = store - .repair(&addr, b"not this chunk") - .await - .expect_err("must refuse"); - assert!(format!("{err}").contains("Refusing to repair"), "{err}"); - assert!( - path.exists(), - "the existing file must survive a refused repair" - ); - assert_eq!( - store.get(&addr).await.expect("get").expect("present"), - content - ); - } - - #[tokio::test] - async fn a_chunk_can_be_deleted_and_stored_again() { - let (store, _dir) = test_store().await; - let (addr, content) = addressed("cycle"); - - assert!(store.put(&addr, &content).await.expect("put")); - assert!(store.delete(&addr).await.expect("delete")); - assert!( - store.put(&addr, &content).await.expect("re-put"), - "a re-stored chunk is new again" - ); - assert_eq!( - store.get(&addr).await.expect("get").expect("present"), - content - ); - } - - /// Write a chunk file straight into its shard, the way an existing store already - /// contains thousands of them. Bypasses the write path deliberately: this exercises - /// the startup scan, not `put`. - fn plant(chunks_dir: &Path, key: &XorName) { - let dir = chunks_dir.join(shard_name(key)); - std::fs::create_dir_all(&dir).expect("mkdir"); - std::fs::write(dir.join(hex::encode(key)), key).expect("plant"); - } - - #[tokio::test] - async fn a_populated_and_churned_store_scans_correctly_at_scale() { - // Every shard populated, then aged the way a long-lived node ages: some keys - // deleted, others added in their place, so the directories carry holes rather - // than being freshly written. APFS enumeration is known to degrade with churn - // rather than with size, so a fresh corpus is not a realistic one. - const PLANTED: u32 = 20_000; - const CHURN: u32 = 1_000; - - let dir = TempDir::new().expect("temp dir"); - let chunks_dir = dir.path().join(CHUNKS_DIR_NAME); - std::fs::create_dir_all(&chunks_dir).expect("mkdir"); - - let mut expected: Vec = Vec::new(); - for i in 0..PLANTED { - let key = crate::client::compute_address(&i.to_le_bytes()); - plant(&chunks_dir, &key); - expected.push(key); - } - for i in 0..CHURN { - let key = crate::client::compute_address(&i.to_le_bytes()); - std::fs::remove_file(chunks_dir.join(shard_name(&key)).join(hex::encode(key))) - .expect("churn out"); - let replacement = crate::client::compute_address(&(PLANTED + i).to_le_bytes()); - plant(&chunks_dir, &replacement); - } - expected.retain(|k| chunks_dir.join(shard_name(k)).join(hex::encode(k)).exists()); - for i in 0..CHURN { - expected.push(crate::client::compute_address(&(PLANTED + i).to_le_bytes())); - } - expected.sort_unstable(); - expected.dedup(); - - let started = std::time::Instant::now(); - let store = reopen(&dir).await; - let scan = started.elapsed(); - - assert_eq!( - store.current_chunks().expect("count"), - expected.len() as u64 - ); - assert_eq!(store.all_keys().await.expect("all_keys"), expected); - - // Every shard should be in use at this size: 20,000 keys over 256 directories is - // about 78 each, and the last byte of a BLAKE3 output is uniform. - let occupied = std::fs::read_dir(&chunks_dir) - .expect("read store root") - .filter_map(std::result::Result::ok) - .filter(|e| e.file_name().to_str().is_some_and(|n| n.len() == 2)) - .count(); - assert_eq!(occupied, SHARD_COUNT, "the suffix must reach every shard"); - - println!( - "scan of {} keys across {SHARD_COUNT} shards took {scan:?}", - expected.len() - ); - } - - #[tokio::test] - async fn wait_idle_returns_once_writes_have_drained() { - let (store, _dir) = test_store().await; - let store = Arc::new(store); - for i in 0..32 { - let store = Arc::clone(&store); - let (addr, content) = addressed(&format!("drain-{i}")); - tokio::spawn(async move { store.put(&addr, &content).await }); - } - // Not a synchronisation point for tasks that have not been spawned yet, but it - // must not hang and it must leave the store usable. - store.wait_idle().await; - let (addr, content) = addressed("after-drain"); - assert!(store.put(&addr, &content).await.expect("put after drain")); - } -} diff --git a/src/storage/handler.rs b/src/storage/handler.rs index 840a5ef9..06b34810 100644 --- a/src/storage/handler.rs +++ b/src/storage/handler.rs @@ -1,7 +1,7 @@ //! ANT protocol handler for autonomi protocol messages. //! //! This handler processes chunk PUT/GET requests with optional payment verification, -//! storing chunks to LMDB and using the DHT for network-wide retrieval. +//! storing chunks on disk and using the DHT for network-wide retrieval. //! //! # Architecture //! @@ -327,10 +327,10 @@ fn settlement_gate(client_settlement_version: u32, path: &str) -> Option, /// Payment verifier for checking payments. payment_verifier: Arc, @@ -356,7 +356,7 @@ impl AntProtocol { /// /// # Arguments /// - /// * `storage` - LMDB storage for chunk persistence + /// * `storage` - the chunk store /// * `payment_verifier` - Payment verifier for validating payments /// * `quote_generator` - Quote generator for creating storage quotes #[must_use] @@ -436,7 +436,7 @@ impl AntProtocol { CHUNK_PROTOCOL_ID } - /// Get a reference to the underlying LMDB storage. + /// Get a reference to the underlying chunk store. #[must_use] pub fn storage(&self) -> Arc { Arc::clone(&self.storage) @@ -922,13 +922,13 @@ impl AntProtocol { /// actually holds. /// /// The quote price is driven by `QuoteGenerator::records_stored()`. Reading - /// the live LMDB entry count (an O(1) B-tree page-header read) right before + /// the live chunk count right before /// pricing makes the metric deletion-aware: any chunk removed by /// [`ChunkStore::delete`] or by the replication prune pass is reflected /// immediately, with no risk of missing a delete path. /// /// On a storage read error — or a count that does not fit `usize` — the - /// previous metric value is left untouched so a transient LMDB error never + /// previous metric value is left untouched so a transient read error never /// disrupts quote generation. fn resync_quote_metric(&self) { match self.storage.current_chunks() { @@ -1567,7 +1567,7 @@ mod tests { /// "Full" now means both halves of the predicate: the volume is below the /// reserve **and** the store has no reusable space. A freshly created store /// has no freed pages, so both hold and the pre-check short-circuits, as it - /// always did. The companion cases in `storage::lmdb::tests` cover the half + /// always did. The companion cases in `storage::chunk_store::tests` cover the half /// that changed, where pruning has left reusable pages and the node must be /// admitted rather than refused on `statvfs` alone. /// diff --git a/src/storage/legacy_artifacts.rs b/src/storage/legacy_artifacts.rs new file mode 100644 index 00000000..d987fbd2 --- /dev/null +++ b/src/storage/legacy_artifacts.rs @@ -0,0 +1,1108 @@ +//! What a node does when it finds the old chunk store still on disk. +//! +//! Chunks used to live in an LMDB environment at `{root}/chunks.mdb`. The previous release +//! copied them into a file per chunk and deleted that environment; this build has no code +//! that can read one. So a node starting with something still there needs an answer, and +//! there are three plausible ones. Two of them are wrong. +//! +//! **Refusing to start is wrong.** It was the first answer, and the reasoning was not silly: +//! those chunks are unreachable, so the node serves less than its published commitment +//! claims, and it spends the answerability window failing commitment-bound audits for keys +//! it cannot read. But a node that refuses serves *nothing* — not the chunks it cannot read, +//! and not the far larger number it migrated perfectly well. Nor can it be recovered: +//! `build_upgrade_monitor` is called unconditionally and `UpgradeConfig` has no field that +//! switches it off, so a node put back on the previous release is dragged forward again +//! within the hour, and on the deployed unit it is a ten-second restart loop until a person +//! intervenes. A few hours of trust penalty avoided, paid for with the whole node, +//! indefinitely. +//! +//! **Deleting whatever is there is also wrong**, and more obviously so once written down. +//! The upgrade monitor picks the newest eligible release rather than the next one, so a node +//! that was offline through the previous release arrives here with every chunk it owns in +//! that environment and nothing in the file store. Deleting it destroys data that may have +//! no other copy, to reclaim disk. +//! +//! So: **start, and remove only what is provably finished with.** The previous release wrote +//! a `RETIRED` mark inside the directory before it deleted anything, so a directory carrying +//! that mark is one whose retirement gates were all met and is pure cost. So is an empty one, +//! which is what a cleanup interrupted between emptying a tombstone and removing it leaves. +//! Those go, and their disk comes back. Anything else stays exactly where it is, and is +//! named once so an operator can decide. +//! +//! "Gates were met" is not "its contents are in the file store", and the difference is load +//! bearing. Retirement cleared a directory on two grounds: every chunk the node KEPT was copied +//! into the file store and re-hashed there, and every chunk it SHED was proven held by its close +//! group — all but one answering a possession challenge — and then deliberately not copied. So a +//! marked directory can legitimately hold bytes that are in no file store on this node. Removing +//! them is finishing what the previous release had already started; the safety argument for them +//! is the close group's proofs, not a local copy. +//! +//! **What may be deleted is decided by [`migration_signal::classify`], not by a second +//! reading of the same directory.** The previous release put each node's answer on the wire +//! so a fleet could be seen to have finished, and this release is published on the strength +//! of that count. If the cleanup had its own notion of which directories are finished with, +//! the two could drift, and a node could delete a directory it was still reporting as +//! unfinished, or report `files` while keeping one. They are one function, and the names +//! they match are one set of constants. +//! +//! Two things are never done, both because a name is not evidence of what is behind it. A +//! link is neither followed nor unlinked: what is behind it is on storage this node does not +//! own. And only the exact names the previous release created are considered at all. + +use std::path::{Path, PathBuf}; + +use crate::logging::{info, warn}; +use crate::storage::migration_signal::{classify, legacy_directories, Leftover, RETIRED_MARKER}; + +/// Remove what the storage migration finished with, and start either way. +/// +/// Returns nothing and fails at nothing. **This is what never vetoes a start** — it is not a +/// claim that every node starts, which is not this module's to make: the file store is built +/// before this runs and a store that cannot open still stops the node. What is ruled out is +/// a node kept from running by what it found left over, which is what the first draft of this +/// release did. +/// +/// Called once the file store has opened, and not before. "Finished with" means the chunks +/// are in the file store, which is only true if the file store is there to hold them: running +/// this earlier put the deletion in front of a constructor that can still fail, and a node +/// that lost both stores that way had nothing to go back to. A node that never opens a store +/// at all does not call this. +/// +/// This runs after the user agent that carries this node's migration state to every peer has +/// already been fixed, and that turns out not to matter: everything removed here is a +/// leftover the signal already calls finished with, because carrying the mark or being empty +/// is exactly what makes it removable and exactly what makes it harmless. The node announces +/// `files` either way, whether the deletion has finished or not yet started. +pub fn clean_up(root_dir: &Path) { + // A root that cannot be listed hides every tombstone under it. The previous release's + // own reporting calls that `unknown` rather than `files`, and the matching answer here is + // to remove nothing and say so: an unlistable root is not evidence that there is nothing + // to keep. Said out loud because an earlier version returned silently, which left the + // decision record promising a warning that no code emitted. + let Ok(leftovers) = legacy_directories(root_dir) else { + warn!( + migration_event = "legacy_store_left", + "{} could not be listed, so nothing left over from the storage migration was \ + removed from it. Any leftover is still costing disk.", + root_dir.display() + ); + return; + }; + + let mut finished_with = Vec::new(); + for dir in leftovers { + match classify(&dir) { + // Its chunks are in the file store and the previous release simply did not + // finish deleting it, or there is nothing in it at all. Either way it holds + // nothing, so removing it cannot lose anything. + Leftover::Harmless => finished_with.push(dir), + Leftover::Holding | Leftover::Unreadable => warn!( + migration_event = "legacy_store_left", + "{} is left over from the storage migration and this build will not remove \ + it: {}. It is costing disk until it is dealt with by hand.", + dir.display(), + why_it_is_kept(&dir) + ), + } + } + + if !finished_with.is_empty() { + remove_all(finished_with); + } +} + +/// Why one directory is being kept, for the operator who has to decide what to do with it. +/// +/// Prose only. The decision was already made by `classify`; this re-reads the directory +/// solely to say which of its reasons applied. If the disk changes underneath the two, the +/// cost is a warning that names the wrong reason, never a directory deleted that should not +/// have been. +/// +/// Its only caller is inside a `warn!`, which compiles to nothing without the `logging` +/// feature, so off that feature this has no caller at all. Same treatment as the signal's own +/// token helpers rather than a `#[cfg]`, which would take the function out of the build the +/// tests run in. +#[cfg_attr(not(feature = "logging"), allow(dead_code))] +fn why_it_is_kept(dir: &Path) -> &'static str { + let Ok(meta) = std::fs::symlink_metadata(dir) else { + return "it cannot be examined"; + }; + if meta.file_type().is_symlink() { + return "it is a link to storage this node does not own"; + } + if !meta.is_dir() { + return "it is not a directory"; + } + match std::fs::symlink_metadata(dir.join(RETIRED_MARKER)) { + Ok(meta) if meta.is_file() => "it carries the mark that says it was finished with", + Ok(_) => { + "something that is not the storage migration's mark is using the name the mark \ + would have, so this node cannot tell whether it was finished with" + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + "it has chunks in it that were never copied into the file store, and this build \ + cannot read them. Nothing here will delete them, so they are not lost. Whether \ + this node's close group still needs them is the question to answer before \ + removing it by hand" + } + Err(_) => "whether it was finished with cannot be established", + } +} + +/// Delete every directory retirement had already cleared, one after another. +/// +/// Not "every directory that holds no chunks": a marked one can hold the chunks the node shed, +/// which retirement proved the close group holds and deliberately did not copy here. +/// +/// **One thread, in sequence.** The names this release recognises are the live directory, the +/// unnumbered tombstone and sixty-four numbered ones, so a root that has been through enough +/// restore cycles can present sixty-six at once. A thread each would put sixty-six concurrent +/// recursive deletions on the disk that is also serving chunks, at the moment a node is +/// starting up. They are pure disk work with nothing waiting on them, so doing them in turn +/// costs nothing that matters and bounds what this can do to a node's I/O. +/// +/// In the background and in place. Nothing has to be got out of the way first: these +/// directories hold nothing, and this build has no code that would read them if they did. Not +/// renaming also means no name to allocate, which is what an earlier version of this could +/// run out of and wedge itself on. +fn remove_all(dirs: Vec) { + let spawned = std::thread::Builder::new() + .name("legacy-store-cleanup".into()) + .spawn(move || { + for dir in dirs { + match delete_mark_last(&dir) { + // Said only once the deletion has finished. Announcing the space before + // it is back is how an operator comes to trust a number that is wrong for + // the next several minutes. + Ok(()) => info!( + migration_event = "space_returned", + "Removed {}, which the storage migration had finished with, and \ + returned its space.", + dir.display() + ), + Err(e) => warn!( + migration_event = "legacy_store_left", + "{} was finished with by the storage migration but could not be \ + removed ({e}). It is costing disk. The next start tries again.", + dir.display() + ), + } + } + }); + if let Err(e) = spawned { + // Naming what did not happen, not just that something did not. An earlier version + // said only that a thread could not start, which tells an operator nothing about + // which disk stayed full. + warn!( + migration_event = "legacy_store_left", + "Could not start the storage migration's cleanup thread ({e}), so nothing it had \ + finished with was removed. Those directories are still costing disk and the next \ + start tries again." + ); + } +} + +/// The most entries this will look at in one leftover. +/// +/// A retired chunk environment holds `data.mdb`, `lock.mdb` and the mark. Something with +/// thousands of entries in it is not one, and reading a corrupt or hostile directory's whole +/// listing into memory is how a cleanup takes the node down with it instead of returning some +/// disk. Over this, the directory is kept and named, which is what happens to everything else +/// this cannot make sense of. +const MAX_ENTRIES: usize = 1024; + +/// Empty a directory, then remove its mark, then remove the directory. +/// +/// The order is the whole of it. `remove_dir_all` gives no promise about which entry it +/// unlinks first, and the mark is the only thing that says this directory was finished with: +/// if it goes before the contents do and the process stops there, the next start finds an +/// unmarked directory with data in it, decides it might be an unmigrated store, and keeps it +/// forever. It is not one, but nothing on disk says so any more, and the node then reports +/// itself as unfinished to the whole network for as long as it lives. +/// +/// **On Unix the directory is opened once and never named again.** An earlier version checked +/// `symlink_metadata(dir)` and then re-opened the same path with `read_dir`, which follows a +/// link. Anything that could replace the directory between the two — a local actor with write +/// access to the node's data directory — could point it at a target elsewhere on the disk and +/// have this process delete that target's contents instead. The node can reach more of the +/// filesystem than such an actor can, so that is not one more way to lose data inside the data +/// directory: it is a way to reach outside it. One handle, opened `O_NOFOLLOW | O_DIRECTORY`, +/// carries every gate and every unlink including the mark's, so the answer that authorises the +/// deletion and the deletion itself are about the same inode by construction. A second open +/// for the mark was a second window and is gone. +/// +/// **Subdirectories are refused, not entered.** `O_NOFOLLOW` declines a trailing symlink; it +/// says nothing about a mount point, and `openat` walks into one happily. A retired chunk +/// environment is flat — two files and a mark — so a directory inside one is already something +/// this build does not understand, and descending into it risks emptying a filesystem that has +/// nothing to do with this node. Refusing also means there is no recursion to bound, no +/// descriptor chain to exhaust, and no cycle a bind mount could build. +/// +/// The directory itself still goes by path, and that is safe on its own terms: `rmdir` refuses +/// a symlink, so a swapped name fails the call rather than following it. It does not preserve +/// identity — a directory swapped for another real directory would be the one removed — but by +/// then the only thing that can be removed is an empty directory, and removing an empty +/// directory somebody else just put there costs nothing. +/// +/// Off Unix there is no `unlinkat`. That build keeps the path-based version, with the same +/// refusals and the same re-check, and the window the previous release's deleter also had. +fn delete_mark_last(dir: &Path) -> std::io::Result<()> { + empty_and_unmark(dir)?; + // `rmdir`, not `unlink`: it refuses a symlink outright, so this one path cannot be followed + // anywhere even if the name were swapped after the emptying. + std::fs::remove_dir(dir) +} + +/// The gates, the contents and the mark, all against one directory handle. +#[cfg(unix)] +fn empty_and_unmark(dir: &Path) -> std::io::Result<()> { + use rustix::fs::{AtFlags, FileType}; + + let dirfd = open_dir_nofollow(dir)?; + + // Asked again, here, on the thread that does the deleting, and asked of the handle. The + // classification that got this path happened on another thread and is already in the past. + // + // A REGULAR file, and the comparison masks with `S_IFMT` to say so. Testing + // `st_mode & S_IFREG == S_IFREG` is not that test: `S_IFLNK` and `S_IFSOCK` both contain + // every bit of `S_IFREG`, so a symlink or a socket wearing the mark's name would pass it + // and authorise deleting a store nothing had migrated. + let marked = matches!( + rustix::fs::statat(&dirfd, RETIRED_MARKER, AtFlags::SYMLINK_NOFOLLOW), + Ok(stat) if FileType::from_raw_mode(stat.st_mode) == FileType::RegularFile + ); + + let names = read_names(&dirfd)?; + if !marked + && names + .iter() + .any(|name| name.as_bytes() != RETIRED_MARKER.as_bytes()) + { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "it no longer carries the mark that said it was finished with, and it is not \ + empty either", + )); + } + + for name in &names { + if name.as_bytes() == RETIRED_MARKER.as_bytes() { + continue; + } + match rustix::fs::unlinkat(&dirfd, name.as_c_str(), AtFlags::empty()) { + // Gone already is the outcome this wanted. + Ok(()) | Err(rustix::io::Errno::NOENT) => {} + // A directory: EISDIR on Linux, EPERM on the BSDs and macOS. Both answers are + // about the inode the handle names, so neither can have been redirected — and + // neither is entered. + Err(rustix::io::Errno::ISDIR | rustix::io::Errno::PERM) => { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "it has a directory inside it, which a retired chunk environment never \ + has. Whatever it is, this build will not walk into it", + )) + } + Err(e) => return Err(e.into()), + } + } + + // Asked once more, on the same handle, before the one irreversible step. `read_names` is a + // listing, not a snapshot: anything created while the loop above ran is not in it and is + // still here. Removing the mark over it is exactly the state the ordering exists to + // prevent — an unmarked directory with something in it, which every later start keeps. + if !is_empty_but_for_the_mark(&dirfd)? { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "something was created in it while it was being emptied, so the mark stays and \ + the next start looks again", + )); + } + + // Now, and only now, the thing that said it was safe to do any of the above. + match rustix::fs::unlinkat(&dirfd, RETIRED_MARKER, AtFlags::empty()) { + // Already gone is success: the directory qualified by being empty, or something else + // finished this. + Ok(()) | Err(rustix::io::Errno::NOENT) => Ok(()), + Err(e) => Err(e.into()), + } +} + +/// Open a directory, refusing a link or anything that is not one, on the handle. +#[cfg(unix)] +fn open_dir_nofollow(dir: &Path) -> std::io::Result { + use rustix::fs::{Mode, OFlags}; + + rustix::fs::open( + dir, + OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC, + Mode::empty(), + ) + .map_err(|e| { + // A link, or something that is not a directory at all, arrives here as ELOOP or + // ENOTDIR. Both mean the same thing to this caller and it is worth saying which. + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!( + "{} is no longer the directory that was found to be finished with ({e})", + dir.display() + ), + ) + }) +} + +/// Is there anything in here besides the mark? +#[cfg(unix)] +fn is_empty_but_for_the_mark(dirfd: &std::os::fd::OwnedFd) -> std::io::Result { + Ok(read_names(dirfd)? + .iter() + .all(|name| name.as_bytes() == RETIRED_MARKER.as_bytes())) +} + +/// Every name in a directory handle, without `.` and `..`, and never more than a leftover has. +/// +/// Collected rather than streamed, because unlinking while a directory stream is open is not +/// something any filesystem promises to keep coherent. It is a listing and not a snapshot, so +/// the caller asks again before the step it cannot take back. +#[cfg(unix)] +fn read_names(dirfd: &std::os::fd::OwnedFd) -> std::io::Result> { + let mut names = Vec::new(); + let dir = rustix::fs::Dir::read_from(dirfd)?; + for entry in dir { + let entry = entry?; + let name = entry.file_name(); + if name.to_bytes() == b"." || name.to_bytes() == b".." { + continue; + } + if names.len() >= MAX_ENTRIES { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!( + "it holds more than {MAX_ENTRIES} entries, which a retired chunk \ + environment never does" + ), + )); + } + names.push(name.to_owned()); + } + Ok(names) +} + +/// The path-based version, for the platforms with no `unlinkat`. +/// +/// The same refusals and the same re-check, so the two do not drift on anything but the window +/// between naming a path and acting on it, which is what `unlinkat` closes and this cannot. +#[cfg(not(unix))] +fn empty_and_unmark(dir: &Path) -> std::io::Result<()> { + if !matches!(std::fs::symlink_metadata(dir), Ok(meta) if meta.is_dir()) { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "it is no longer the directory that was found to be finished with", + )); + } + let marked = matches!( + std::fs::symlink_metadata(dir.join(RETIRED_MARKER)), + Ok(meta) if meta.is_file() + ); + let names = read_names_by_path(dir)?; + if !marked && names.iter().any(|name| name != RETIRED_MARKER) { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "it no longer carries the mark that said it was finished with, and it is not \ + empty either", + )); + } + for name in &names { + if name == RETIRED_MARKER { + continue; + } + let path = dir.join(name); + if matches!(std::fs::symlink_metadata(&path), Ok(meta) if meta.is_dir()) { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "it has a directory inside it, which a retired chunk environment never has. \ + Whatever it is, this build will not walk into it", + )); + } + match std::fs::remove_file(&path) { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => return Err(e), + } + } + if read_names_by_path(dir)? + .iter() + .any(|name| name != RETIRED_MARKER) + { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "something was created in it while it was being emptied, so the mark stays and \ + the next start looks again", + )); + } + match std::fs::remove_file(dir.join(RETIRED_MARKER)) { + Ok(()) => Ok(()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(e), + } +} + +/// Every name in a directory, and never more than a leftover has. +#[cfg(not(unix))] +fn read_names_by_path(dir: &Path) -> std::io::Result> { + let mut names = Vec::new(); + for entry in std::fs::read_dir(dir)? { + if names.len() >= MAX_ENTRIES { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!( + "it holds more than {MAX_ENTRIES} entries, which a retired chunk \ + environment never does" + ), + )); + } + names.push(entry?.file_name()); + } + Ok(names) +} + +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +mod tests { + use super::*; + use crate::storage::migration_signal::{is_tombstone_name, LEGACY_ENV_DIR, MAX_TOMBSTONES}; + use tempfile::TempDir; + + /// The deletion runs on its own thread. + fn wait_gone(path: &Path) -> bool { + for _ in 0..200 { + if !path.exists() { + return true; + } + std::thread::sleep(std::time::Duration::from_millis(10)); + } + false + } + + fn settle() { + std::thread::sleep(std::time::Duration::from_millis(200)); + } + + fn env_with_chunks(root: &Path, name: &str) -> PathBuf { + let dir = root.join(name); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("data.mdb"), b"chunks").unwrap(); + dir + } + + /// A socket at `path`, for the one type test that needs one. + #[cfg(unix)] + fn bind_socket(path: &Path) { + std::os::unix::net::UnixListener::bind(path).expect("bind"); + } + + fn mark_retired(dir: &Path) { + std::fs::write(dir.join(RETIRED_MARKER), b"retired").unwrap(); + } + + /// The property this module exists for: whatever is on disk, the node runs. + #[test] + fn nothing_here_can_stop_a_node_starting() { + let root = TempDir::new().unwrap(); + env_with_chunks(root.path(), LEGACY_ENV_DIR); + env_with_chunks(root.path(), "chunks.mdb.retired.2"); + std::fs::create_dir_all(root.path().join("chunks.mdb.retired")).unwrap(); + + // No Result to unwrap: there is no way for this to refuse. + clean_up(root.path()); + settle(); + } + + #[test] + fn a_root_that_does_not_exist_yet_is_fine() { + let root = TempDir::new().unwrap(); + clean_up(&root.path().join("not").join("created")); + } + + /// The one this release is for: a store the migration finished with but did not delete. + #[test] + fn a_store_the_migration_finished_with_is_removed() { + let root = TempDir::new().unwrap(); + let env = env_with_chunks(root.path(), LEGACY_ENV_DIR); + mark_retired(&env); + clean_up(root.path()); + assert!(wait_gone(&env)); + } + + #[test] + fn a_marked_tombstone_is_removed_too() { + let root = TempDir::new().unwrap(); + let tomb = env_with_chunks(root.path(), "chunks.mdb.retired.7"); + mark_retired(&tomb); + clean_up(root.path()); + assert!(wait_gone(&tomb)); + } + + #[test] + fn an_empty_leftover_is_removed() { + // What a cleanup interrupted between emptying a tombstone and removing it leaves. + let root = TempDir::new().unwrap(); + let tomb = root.path().join("chunks.mdb.retired"); + std::fs::create_dir_all(&tomb).unwrap(); + clean_up(root.path()); + assert!(wait_gone(&tomb)); + } + + /// A marked directory goes even though its bytes are in no file store on this node. + /// + /// This is the case the record used to describe wrongly, and the tests used to conceal by + /// never staging a file store at all: they showed that a mark triggers deletion, which + /// leaves a reader free to assume the deleted bytes had been copied somewhere first. + /// + /// They need not have been. Retirement cleared a directory on two grounds, and only one is + /// a local copy: chunks the node KEPT were copied into the file store and re-hashed there, + /// and chunks it SHED were proven held by its close group and then deliberately not copied + /// — the pre-retirement pass skips exactly those keys. So the state staged here, a marked + /// directory beside an empty file store, is one the previous release produces on purpose on + /// any node that was short of disk, and it is reachable through its own kill point between + /// writing the mark and finishing the delete. + /// + /// Removing it is right: the previous release was about to. What this pins is that the + /// deletion does NOT depend on a local copy existing, so nobody later "fixes" it by adding + /// a file-store check that would strand every shedding node's directory for ever. + #[test] + fn a_marked_store_goes_even_when_nothing_was_copied_into_this_node() { + let root = TempDir::new().unwrap(); + let env = env_with_chunks(root.path(), LEGACY_ENV_DIR); + mark_retired(&env); + + // No file store, and nothing in one. A node that shed everything it held looks exactly + // like this. + assert!(!root.path().join("chunks").exists()); + + clean_up(root.path()); + assert!( + wait_gone(&env), + "a marked directory must go whether or not this node kept a copy: the close \ + group's possession proofs are what cleared it, not a local file" + ); + } + + /// The one that would be data loss. + /// + /// A node that missed the previous release entirely arrives here with every chunk it + /// owns in that directory and nothing in the file store, and the upgrade monitor picks + /// the newest release rather than the next one, so this build is genuinely reachable + /// from that state. Unmarked is the whole difference from the test above: nothing + /// cleared this directory, so nothing here may remove it. + #[test] + fn a_store_that_was_never_migrated_is_left_exactly_where_it_is() { + let root = TempDir::new().unwrap(); + let env = env_with_chunks(root.path(), LEGACY_ENV_DIR); + clean_up(root.path()); + settle(); + assert!( + env.join("data.mdb").exists(), + "an unmarked store passed no retirement gate, and was deleted anyway" + ); + } + + #[test] + fn an_unmarked_tombstone_with_chunks_in_it_is_left_too() { + // A crash between the rename and the mark leaves an intact environment wearing a + // retired-looking name. What it is called is not evidence. + let root = TempDir::new().unwrap(); + let tomb = env_with_chunks(root.path(), "chunks.mdb.retired.3"); + clean_up(root.path()); + settle(); + assert!(tomb.join("data.mdb").exists()); + } + + /// A name is not evidence, and this decides what gets deleted. + #[test] + fn a_directory_that_only_looks_like_a_tombstone_is_left_alone() { + let root = TempDir::new().unwrap(); + let mine = env_with_chunks(root.path(), "chunks.mdb.retired-keep-this"); + mark_retired(&mine); + let also = env_with_chunks(root.path(), "chunks.mdb.backup"); + mark_retired(&also); + let high = env_with_chunks(root.path(), "chunks.mdb.retired.65"); + mark_retired(&high); + let padded = env_with_chunks(root.path(), "chunks.mdb.retired.007"); + mark_retired(&padded); + + clean_up(root.path()); + settle(); + for kept in [&mine, &also, &high, &padded] { + assert!( + kept.exists(), + "{} was deleted and nothing here created it", + kept.display() + ); + } + + assert!(is_tombstone_name("chunks.mdb.retired")); + assert!(is_tombstone_name("chunks.mdb.retired.1")); + assert!(is_tombstone_name("chunks.mdb.retired.64")); + assert!(!is_tombstone_name("chunks.mdb.retired.65")); + assert!(!is_tombstone_name("chunks.mdb.retired.0")); + assert!(!is_tombstone_name("chunks.mdb.retired.007")); + assert!(!is_tombstone_name("chunks.mdb.retired-keep-this")); + assert!(!is_tombstone_name("chunks.mdb.retired.")); + assert!(!is_tombstone_name("chunks.mdb.retired.x")); + } + + /// The name is reserved, but only a file R2 could have written is evidence. + #[test] + fn something_else_wearing_the_marks_name_is_not_the_mark() { + let root = TempDir::new().unwrap(); + let env = env_with_chunks(root.path(), LEGACY_ENV_DIR); + // A directory at the reserved name, which `try_exists` cannot tell from the file + // retirement writes. Believing it deletes an unmigrated store. + std::fs::create_dir_all(env.join(RETIRED_MARKER)).unwrap(); + + clean_up(root.path()); + settle(); + assert!( + env.join("data.mdb").exists(), + "a directory called RETIRED authorised deleting an unmigrated store" + ); + } + + /// The mark is removed last, so an interrupted deletion is still recognisable. + /// + /// `remove_dir_all` promises nothing about order. If the mark went first and the process + /// stopped there, the next start would find an unmarked directory with data in it, treat + /// it as a store that was never migrated, and keep it for good — and the node would + /// report itself unfinished to the whole network for as long as it lived. + #[test] + fn the_mark_outlives_everything_it_was_vouching_for() { + let root = TempDir::new().unwrap(); + let env = env_with_chunks(root.path(), LEGACY_ENV_DIR); + std::fs::write(env.join("lock.mdb"), b"lock").unwrap(); + mark_retired(&env); + + // The deletion, run inline so its intermediate states can be inspected rather than + // raced. + delete_mark_last(&env).unwrap(); + assert!(!env.exists()); + + // And the order it does it in: with the directory made unremovable, the mark has to + // still be there when the attempt fails. + let other = TempDir::new().unwrap(); + let env = env_with_chunks(other.path(), LEGACY_ENV_DIR); + mark_retired(&env); + let nested = env.join("sub"); + std::fs::create_dir_all(&nested).unwrap(); + std::fs::write(nested.join("data.mdb"), b"chunks").unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&nested, std::fs::Permissions::from_mode(0o500)).unwrap(); + let failed = delete_mark_last(&env); + std::fs::set_permissions(&nested, std::fs::Permissions::from_mode(0o755)).unwrap(); + assert!(failed.is_err(), "the deletion was supposed to fail here"); + assert!( + env.join(RETIRED_MARKER).exists(), + "the mark went before the chunks did, so the next start cannot tell this \ + directory was already finished with" + ); + } + } + + /// The deleting thread asks again rather than trusting a path it was handed. + /// + /// What is at a path is not a property of the path. The classification happened on + /// another thread and is already in the past, so between the two moments the directory + /// can be replaced with a link to somewhere else, and following it would delete through + /// it. Staged directly here because a real race is not reproducible in a test. + #[cfg(unix)] + #[test] + fn the_deleting_thread_refuses_a_path_that_is_no_longer_what_was_classified() { + let root = TempDir::new().unwrap(); + let elsewhere = env_with_chunks(root.path(), "elsewhere"); + // MARKED, and that is the whole point of this test. An unmarked target would be + // refused by the gates even by a deleter that happily followed the link, so the test + // would pass while proving nothing about following it. Marked, the target satisfies + // every gate: anything that reaches it through the link deletes it. + mark_retired(&elsewhere); + let swapped = root.path().join(LEGACY_ENV_DIR); + std::os::unix::fs::symlink(&elsewhere, &swapped).unwrap(); + + assert!( + delete_mark_last(&swapped).is_err(), + "the deletion followed a link that appeared after the classification" + ); + assert!( + elsewhere.join("data.mdb").exists(), + "the link was followed and the contents of somewhere else were deleted" + ); + assert!(elsewhere.join(RETIRED_MARKER).exists()); + + // And a directory that lost its mark in between: it might be a store nothing + // migrated, so it is not deleted on the strength of an answer given earlier. + let unmarked = env_with_chunks(root.path(), "chunks.mdb.retired.9"); + assert!(delete_mark_last(&unmarked).is_err()); + assert!(unmarked.join("data.mdb").exists()); + } + + /// A link never yields a directory handle, which is what leaves no window to win. + /// + /// The test above stages a link that is already there when the deletion starts, and the + /// path-based version caught that too, by asking `symlink_metadata` first. What it could + /// not catch is a link that arrives *between* that question and the `read_dir` that acted + /// on the answer: `read_dir` resolves the path again and follows what it finds. A local + /// actor with write access to the node's data directory could point that name at a target + /// elsewhere on the disk, and the node — which reaches more of the filesystem than the + /// actor does, and on plenty of installations runs as root — would empty the target + /// instead. Not one more way to lose data inside the data directory: a way out of it. + /// + /// That interleaving cannot be staged from a test without instrumenting the deleter, so + /// what is pinned here is the primitive that removes the window rather than the race + /// itself. The directory is opened once, and every unlink is made against that handle, so + /// if a link can never produce a handle then no unlink can ever be redirected through one. + /// This is the "can never produce a handle" half, and it is the half that a change would + /// silently undo: drop `O_NOFOLLOW` and the open below starts succeeding. + #[cfg(unix)] + #[test] + fn a_link_never_yields_a_directory_handle() { + let root = TempDir::new().unwrap(); + let real = env_with_chunks(root.path(), "real"); + + assert!( + open_dir_nofollow(&real).is_ok(), + "a directory that is what it says it is has to open, or nothing is ever cleaned up" + ); + + let link = root.path().join("link"); + std::os::unix::fs::symlink(&real, &link).unwrap(); + assert!( + open_dir_nofollow(&link).is_err(), + "a link produced a handle, so every unlink made against it lands on the target" + ); + + // The other thing that must never be mistaken for a store. A FIFO would block a + // plain open forever; refusing on the handle refuses it without opening it. + let not_a_dir = root.path().join("file"); + std::fs::write(¬_a_dir, b"x").unwrap(); + assert!(open_dir_nofollow(¬_a_dir).is_err()); + } + + /// Only a regular file is the mark. A link wearing its name is not, and neither is a socket. + /// + /// This is a type test, and the arithmetic is the whole of it. `S_IFLNK` and `S_IFSOCK` each + /// contain every bit of `S_IFREG`, so `st_mode & S_IFREG == S_IFREG` is true for all three + /// and is not a test for a regular file at all. Masking with `S_IFMT` is. + /// + /// What it costs to get wrong: the classifier on the main thread can call a leftover + /// harmless while it is empty, and between that and the worker opening it, a restore or a + /// local actor can fill it with a store nothing migrated and drop a symlink called `RETIRED` + /// beside the contents. The worker asks again — that re-ask is the whole reason it exists — + /// and a mask that accepts a link answers yes, and the store goes. + /// + /// Asserted against `delete_mark_last` rather than against the classifier, because the + /// classifier is not where the re-ask happens and a test on it cannot see this. + #[cfg(unix)] + #[test] + fn only_a_regular_file_is_the_mark_when_the_deleting_thread_asks() { + let root = TempDir::new().unwrap(); + + // A link at the mark's name, pointing at a real regular file so that following it + // would find one. + let linked = env_with_chunks(root.path(), "chunks.mdb.retired.11"); + let real_file = root.path().join("somewhere-else"); + std::fs::write(&real_file, b"not the mark").unwrap(); + std::os::unix::fs::symlink(&real_file, linked.join(RETIRED_MARKER)).unwrap(); + + assert!( + delete_mark_last(&linked).is_err(), + "a symlink was accepted as the mark, which authorises deleting an unmigrated store" + ); + assert!( + linked.join("data.mdb").exists(), + "and the store it was guarding is gone" + ); + + // A socket at the mark's name. Same arithmetic, different type. + let socketed = env_with_chunks(root.path(), "chunks.mdb.retired.12"); + bind_socket(&socketed.join(RETIRED_MARKER)); + assert!( + delete_mark_last(&socketed).is_err(), + "a socket was accepted as the mark" + ); + assert!(socketed.join("data.mdb").exists()); + + // And the real thing still works, or none of the above proves anything. + let genuine = env_with_chunks(root.path(), "chunks.mdb.retired.13"); + mark_retired(&genuine); + assert!(delete_mark_last(&genuine).is_ok()); + assert!(!genuine.exists()); + } + + /// A directory inside a leftover is refused, never entered. + /// + /// `O_NOFOLLOW` declines a trailing symlink and says nothing about a mount point, and + /// `openat` walks into one without complaint. A retired chunk environment is flat — two + /// files and a mark — so a directory inside one is already something this build does not + /// understand, and descending into it could empty a filesystem that has nothing to do with + /// this node. Refusing also leaves no recursion to bound and no cycle a bind mount could + /// build. + #[test] + fn a_directory_inside_a_leftover_is_refused_rather_than_entered() { + let root = TempDir::new().unwrap(); + let env = env_with_chunks(root.path(), LEGACY_ENV_DIR); + mark_retired(&env); + let inside = env.join("somewhere"); + std::fs::create_dir_all(&inside).unwrap(); + std::fs::write(inside.join("not-ours"), b"another filesystem's problem").unwrap(); + + assert!( + delete_mark_last(&env).is_err(), + "a directory inside a leftover was walked into" + ); + assert!( + inside.join("not-ours").exists(), + "and its contents were deleted" + ); + assert!( + env.join(RETIRED_MARKER).exists(), + "the mark has to survive a refusal, or the next start cannot tell what this is" + ); + } + + /// A listing is not a snapshot, so the mark is not removed over something that arrived late. + /// + /// The names are read once and then unlinked one by one. Anything created after that read + /// is not in the list and is still there when the loop finishes. Removing the mark over it + /// leaves an unmarked directory with something in it, which is the exact state the ordering + /// exists to prevent and which every later start then keeps for good. + /// + /// The real interleaving needs a write between the read and the mark's removal; staged here + /// by leaving an entry the loop was never told about, which is the same state it produces. + #[test] + fn the_mark_is_not_removed_over_an_entry_that_arrived_late() { + let root = TempDir::new().unwrap(); + let env = env_with_chunks(root.path(), LEGACY_ENV_DIR); + mark_retired(&env); + + // A name the loop cannot unlink, so the directory is still non-empty when the re-ask + // happens. What matters is the answer to that re-ask, not how the entry got there. + let stubborn = env.join("arrived-late"); + std::fs::create_dir_all(&stubborn).unwrap(); + + assert!(delete_mark_last(&env).is_err()); + assert!( + env.join(RETIRED_MARKER).exists(), + "the mark went while the directory still had something in it, so the next start \ + sees an unmigrated store and keeps it for ever" + ); + } + + /// A directory far wider than any leftover is refused rather than read into memory. + #[test] + fn an_implausibly_wide_leftover_is_refused() { + let root = TempDir::new().unwrap(); + let env = root.path().join(LEGACY_ENV_DIR); + std::fs::create_dir_all(&env).unwrap(); + mark_retired(&env); + for n in 0..(MAX_ENTRIES + 8) { + std::fs::write(env.join(format!("e{n}")), b"x").unwrap(); + } + + assert!( + delete_mark_last(&env).is_err(), + "a directory with more entries than any leftover has was read in full" + ); + assert!(env.join(RETIRED_MARKER).exists()); + } + + /// The cleanup removes exactly what the fleet signal calls finished with, and nothing else. + /// + /// This is the property that lets the previous release's count authorise this one. That + /// count is each node reporting `files` when nothing under its root is `Leftover::Holding` + /// or `Leftover::Unreadable`. If this release deleted anything the signal would not have + /// called harmless, a node could destroy a directory it was still reporting as unfinished; + /// if it kept something the signal called harmless, a node would report `files` and go on + /// paying for the disk for ever. + /// + /// They cannot disagree today because there is one `classify`. This is what fails if + /// somebody gives the cleanup its own again: every shape below is checked both ways round, + /// so a classifier that is wrong in either direction shows up here rather than on a fleet. + #[test] + fn the_cleanup_removes_exactly_what_the_signal_calls_finished_with() { + let root = TempDir::new().unwrap(); + let base = root.path(); + + // Every shape a real root can present, harmless and not, with the near misses that + // are not this release's to touch at all. + let marked_live = env_with_chunks(base, LEGACY_ENV_DIR); + mark_retired(&marked_live); + let marked_tomb = env_with_chunks(base, "chunks.mdb.retired.5"); + mark_retired(&marked_tomb); + let empty_tomb = base.join("chunks.mdb.retired"); + std::fs::create_dir_all(&empty_tomb).unwrap(); + let unmarked_tomb = env_with_chunks(base, "chunks.mdb.retired.6"); + let fake_mark = env_with_chunks(base, "chunks.mdb.retired.7"); + std::fs::create_dir_all(fake_mark.join(RETIRED_MARKER)).unwrap(); + let near_miss = env_with_chunks(base, "chunks.mdb.retired.65"); + mark_retired(&near_miss); + let operators = env_with_chunks(base, "chunks.mdb.retired-keep-this"); + mark_retired(&operators); + + // Not a directory at all, wearing a name in the set. The marker probe cannot even be + // asked of it, so it classifies unreadable and must be kept. + let not_a_dir = base.join("chunks.mdb.retired.8"); + std::fs::write(¬_a_dir, b"not a store").unwrap(); + + // A link wearing a name in the set. What is behind it is somebody else's, and neither + // following it nor unlinking it is this build's decision. + #[cfg(unix)] + let linked = { + let elsewhere = env_with_chunks(base, "elsewhere-for-the-link"); + mark_retired(&elsewhere); + let link = base.join("chunks.mdb.retired.9"); + std::os::unix::fs::symlink(&elsewhere, &link).unwrap(); + (link, elsewhere) + }; + + // What the signal says about each, before anything is removed. Every classification + // the enum can produce is represented: Harmless three ways, Holding two, Unreadable + // two. + // The `mut` is used only where the link below exists, which is not everywhere. + #[cfg_attr(not(unix), allow(unused_mut))] + let mut considered = vec![ + &marked_live, + &marked_tomb, + &empty_tomb, + &unmarked_tomb, + &fake_mark, + ¬_a_dir, + ]; + #[cfg(unix)] + considered.push(&linked.0); + let verdicts: Vec<_> = considered.iter().map(|dir| (*dir, classify(dir))).collect(); + + clean_up(base); + settle(); + + for (dir, verdict) in verdicts { + let gone = !dir.exists(); + assert_eq!( + gone, + verdict == Leftover::Harmless, + "{} was classified {verdict:?} and {} removed", + dir.display(), + if gone { "was" } else { "was not" } + ); + } + + // And the names outside the set are not classified at all, so they are never even + // considered for deletion. + for untouched in [&near_miss, &operators] { + assert!( + untouched.join("data.mdb").exists(), + "{} is not a name retirement can have created", + untouched.display() + ); + } + + // A link is kept as a link, and what it points at keeps its data: following it would + // delete storage this node does not own, unlinking it would throw away the only record + // of where that data went. + #[cfg(unix)] + { + let (link, elsewhere) = &linked; + assert!( + std::fs::symlink_metadata(link).is_ok(), + "the link was removed" + ); + assert!( + elsewhere.join("data.mdb").exists(), + "the link was followed and somebody else's data was deleted" + ); + } + } + + /// Sixty-six leftovers are removed one after another, on one thread. + /// + /// The accepted names are the live directory, the unnumbered tombstone and sixty-four + /// numbered ones. A thread each would put sixty-six recursive deletions on the disk that + /// is also serving chunks, at the moment the node is starting. + #[test] + fn every_leftover_a_root_can_hold_is_removed_without_a_thread_each() { + let root = TempDir::new().unwrap(); + let base = root.path(); + + let mut staged = vec![env_with_chunks(base, LEGACY_ENV_DIR)]; + staged.push(env_with_chunks(base, "chunks.mdb.retired")); + for n in 1..=MAX_TOMBSTONES { + staged.push(env_with_chunks(base, &format!("chunks.mdb.retired.{n}"))); + } + assert_eq!(staged.len(), 66, "the accepted namespace is this big"); + for dir in &staged { + mark_retired(dir); + } + + let before = std::thread::available_parallelism().is_ok(); + clean_up(base); + assert!(before, "sanity: the platform reports its parallelism"); + + for dir in &staged { + assert!( + wait_gone(dir), + "{} was finished with and is still there", + dir.display() + ); + } + } + + /// A root that cannot be listed removes nothing, and says so. + /// + /// An unlistable root hides every tombstone under it, so it is not evidence that there is + /// nothing to keep. An earlier version returned silently here, which left the decision + /// record promising a warning that no code emitted. + #[cfg(unix)] + #[test] + fn a_root_that_cannot_be_listed_removes_nothing() { + use std::os::unix::fs::PermissionsExt; + + let root = TempDir::new().unwrap(); + let base = root.path().join("locked"); + std::fs::create_dir_all(&base).unwrap(); + let env = env_with_chunks(&base, LEGACY_ENV_DIR); + mark_retired(&env); + + std::fs::set_permissions(&base, std::fs::Permissions::from_mode(0o000)).unwrap(); + clean_up(&base); + settle(); + std::fs::set_permissions(&base, std::fs::Permissions::from_mode(0o755)).unwrap(); + + assert!( + env.join("data.mdb").exists(), + "a root that could not be listed had something removed from it anyway" + ); + } + + /// What a link points at belongs to somebody else, and is never followed or removed. + #[cfg(unix)] + #[test] + fn a_linked_environment_is_left_exactly_as_it_is() { + let root = TempDir::new().unwrap(); + let elsewhere = env_with_chunks(root.path(), "elsewhere"); + mark_retired(&elsewhere); + let link = root.path().join(LEGACY_ENV_DIR); + std::os::unix::fs::symlink(&elsewhere, &link).unwrap(); + + clean_up(root.path()); + settle(); + + assert!( + std::fs::symlink_metadata(&link).is_ok(), + "the link itself was removed, which throws away the only record of where the \ + data went" + ); + assert!( + elsewhere.join("data.mdb").exists(), + "the link was followed and somebody else's data was deleted" + ); + } +} diff --git a/src/storage/lmdb.rs b/src/storage/lmdb.rs deleted file mode 100644 index 82b97cca..00000000 --- a/src/storage/lmdb.rs +++ /dev/null @@ -1,2446 +0,0 @@ -//! Content-addressed LMDB storage for chunks. -//! -//! Provides persistent storage for chunks using LMDB (via heed) for -//! memory-mapped, zero-copy reads with ACID transactions. -//! -//! ```text -//! {root}/chunks.mdb/ -- LMDB environment directory -//! ``` - -use crate::ant_protocol::{XorName, MAX_CHUNK_SIZE}; -use crate::error::{Error, Result}; -use crate::logging::{debug, info, trace, warn}; -use heed::types::Bytes; -use heed::{Database, Env, EnvOpenOptions, MdbError}; -use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::sync::Arc; -use std::time::Instant; -use tokio::task::spawn_blocking; -use tokio_util::task::TaskTracker; - -use crate::ant_protocol::XORNAME_LEN; -use crate::storage::StorageStats; - -use crate::storage::{GIB, MIB}; - -/// Default minimum free disk space to preserve on the storage partition. -const DEFAULT_DISK_RESERVE: u64 = 500 * MIB; - -/// Convert a byte count to GiB for human-readable log messages. -#[allow(clippy::cast_precision_loss)] // display only — sub-byte precision is irrelevant -fn bytes_to_gib(bytes: u64) -> f64 { - bytes as f64 / GIB as f64 -} - -/// Absolute minimum LMDB map size. -/// -/// Even on a nearly-full disk the database must be able to open. -/// Set to 256 MiB — enough for millions of LMDB pages. -const MIN_MAP_SIZE: usize = 256 * 1024 * 1024; - -/// Maximum head-room (beyond the current data footprint) to reserve for the -/// LMDB map **on Windows**. -/// -/// On Windows a node's committed / private memory scales with the *mapped* -/// size rather than with the data actually stored. Measured on a live node: -/// ~7.4 GB of extra commit for a ~3.7 TiB map, versus ~181 MB for a ~55 GiB -/// map — roughly 0.2% of the mapped size, resident and attributed to *no* -/// user-space allocation. That size-proportional cost is consistent with -/// kernel page tables / section metadata for the mapping; the exact kernel -/// structure was inferred from the scaling rather than measured directly, but -/// the size-proportional *effect* is what this cap targets. Linux keeps the -/// mapping sparse, so a disk-sized map is nearly free there — the overhead is -/// Windows-specific, which is why it only surfaced in Windows reports. -/// -/// Sizing the map to the whole disk therefore costs ~0.2% of *free disk* per -/// node, multiplied by every node sharing the host (e.g. a 10 TiB partition ≈ -/// 20 GiB). We instead cap the head-room on Windows and lean on -/// `LmdbStorage::try_resize` to extend the map on demand as data accumulates, -/// keeping the overhead proportional to *stored data* rather than *disk -/// capacity*. At 32 GiB the extra commit is ~100 MB, and a resize happens at -/// most once per 32 GiB written. -#[cfg(windows)] -const WINDOWS_MAP_HEADROOM: u64 = 32 * GIB; - -/// How often to re-query available disk space (in seconds). -/// -/// Between checks the cached result is trusted. Disk space changes slowly -/// relative to chunk-write throughput, so a multi-second window is safe. -const DISK_CHECK_INTERVAL_SECS: u64 = 5; - -/// Ceiling raise offered to a single *delete* that cannot copy-on-write inside -/// the pinned map. -/// -/// A delete is itself a write: LMDB copies the B-tree path before it frees the -/// leaf pages, and it may need a page for the free-list's own bookkeeping. On a -/// map pinned exactly to the file size a delete therefore has nowhere to go, -/// and the node could not prune its way back to health. -/// -/// Granted **only** on the delete retry path and taken away again inside the -/// same locked scope, so an ordinary store can never allocate from it. Leaving -/// it permanently in the ceiling would hand every node a little more of the -/// very reserve this mode exists to protect, multiplied by the nodes sharing -/// the volume. -const DELETE_COW_SLACK: u64 = 256 * 1024; - -/// Total permanent file growth deletes may cause per low-disk episode. -/// -/// What actually needs bounding is *growth*, not grants. Most slack-assisted -/// deletes reuse pages already inside `data.mdb` and grow it by nothing, and -/// those must stay free: a node has to be able to prune indefinitely, and page -/// reuse is not reliably available to the very next delete because LMDB cannot -/// hand back pages a still-recent transaction freed. Charging per grant instead -/// of per byte stops a node pruning after its first assisted delete. -/// -/// Only bytes the file actually gained are charged here. Reset when the store -/// leaves no-growth mode. A rounding error against [`DEFAULT_DISK_RESERVE`]. -const DELETE_COW_GROWTH_BUDGET: u64 = 1024 * 1024; - -/// Configuration for LMDB storage. -#[derive(Debug, Clone)] -pub struct LmdbStorageConfig { - /// Root directory for storage (LMDB env lives at `{root_dir}/chunks.mdb/`). - pub root_dir: PathBuf, - /// Whether to verify content on read (compares hash to address). - pub verify_on_read: bool, - /// Explicit LMDB map size cap in bytes. - /// - /// When 0 (default), the map size is computed automatically from available - /// disk space and grows on demand when more storage becomes available. - pub max_map_size: usize, - /// Minimum free disk space (in bytes) to preserve on the storage partition. - /// - /// Writes are refused when available space drops below this threshold. - pub disk_reserve: u64, -} - -impl Default for LmdbStorageConfig { - fn default() -> Self { - Self { - root_dir: PathBuf::from(".ant/chunks"), - verify_on_read: true, - max_map_size: 0, - disk_reserve: DEFAULT_DISK_RESERVE, - } - } -} - -impl LmdbStorageConfig { - /// A test-friendly default with `disk_reserve` set to 0 so unit tests - /// don't depend on the host having >= 1 GiB free disk space. - #[cfg(any(test, feature = "test-utils"))] - #[must_use] - pub fn test_default() -> Self { - Self { - disk_reserve: 0, - ..Self::default() - } - } -} - -/// Content-addressed LMDB storage. -/// -/// Uses heed (LMDB wrapper) for memory-mapped, transactional chunk storage. -/// Keys are 32-byte `XorName` addresses, values are raw chunk bytes. -pub struct LmdbStorage { - /// LMDB environment. - env: Env, - /// The unnamed default database (key=XorName bytes, value=chunk bytes). - db: Database, - /// Storage configuration. - config: LmdbStorageConfig, - /// Path to the LMDB environment directory (for disk-space queries). - env_dir: PathBuf, - /// Operation statistics. - stats: parking_lot::RwLock, - /// Serialises access to the LMDB environment during a map resize. - /// - /// Normal read/write operations acquire a **shared** lock. The rare - /// resize path acquires an **exclusive** lock, ensuring no transactions - /// are active when `env.resize()` is called (an LMDB safety requirement). - env_lock: Arc>, - /// Timestamp of the last successful disk-space check. - /// - /// `None` means "never checked — check on next write". Updated only - /// after a passing check, so a low-space result is always rechecked. - last_disk_ok: parking_lot::Mutex>, - /// Whether the map is currently pinned to the file's high-water mark. - /// - /// Set once available disk drops below the reserve. While pinned, LMDB can - /// still serve a write from its own free list but cannot extend - /// `data.mdb`, so the reserve is preserved by the allocator itself rather - /// than by refusing every write up front. - no_growth: Arc, - /// Keep the map pinned whatever the free space says. - /// - /// Set for the whole of the migration bridge. While both stores are open they each - /// measure the same free space and neither knows what the other is about to spend, so - /// a chunk written to both can be admitted twice against one lot of headroom and the - /// pair can cross the reserve together. Pinned, this environment cannot claim any new - /// disk at all: a write it cannot satisfy from its own free list is refused, and the - /// caller stores the chunk in files alone. That is the right answer anyway, because - /// this copy exists to make a rollback survivable, not to be the one that must - /// succeed. - growth_pinned: Arc, - /// Serialises entering and leaving no-growth mode. - /// - /// Setting `no_growth` and resizing the map is one compound transition - /// spanning an await. Without this, two callers straddling the threshold - /// can interleave so the flag ends up describing a map size that was never - /// applied, leaving the store unpinned while it believes it is pinned. - growth_mode_lock: tokio::sync::Mutex<()>, - /// Bytes `data.mdb` has permanently gained to slack-assisted deletes in - /// this low-disk episode. - /// - /// A delete's copy-on-write can extend the file, and LMDB never gives file - /// space back, so that growth is permanent. Bounding it stops repeated - /// fill-then-delete cycles walking the file into the reserve. Deletes that - /// find room inside the file cost nothing. Reset on leaving no-growth mode. - delete_growth_charged: Arc, - /// Tracks every LMDB blocking task spawned by this storage. - /// - /// A `spawn_blocking` closure owns a cloned [`Env`] and keeps running - /// even when its async awaiter is dropped (e.g. by a `select!` losing to - /// a shutdown token). Tracking the blocking task itself — not the async - /// wrapper — lets [`Self::wait_idle`] wait for true quiescence before - /// the environment may be reopened. - blocking_tracker: TaskTracker, - /// Test-only gate read-acquired at the top of the put blocking closure. - /// - /// Tests hold the write half to deterministically park an in-flight put - /// on the blocking pool (e.g. to prove [`Self::wait_idle`] waits for a - /// detached write). - #[cfg(any(test, feature = "test-utils"))] - test_put_gate: Arc>, - /// Test-only gate read-acquired inside the raw-read blocking closures, - /// immediately after the shared `env_lock` guard is taken. - /// - /// Tests hold the write half to deterministically park an in-flight raw - /// read while it still holds the shared environment lock (e.g. to prove - /// [`Self::try_resize`] waits for active raw reads before calling - /// `env.resize()`). - #[cfg(any(test, feature = "test-utils"))] - test_read_gate: Arc>, -} - -impl LmdbStorage { - /// Create a new LMDB storage instance. - /// - /// Opens (or creates) an LMDB environment at `{root_dir}/chunks.mdb/`. - /// - /// When `config.max_map_size` is 0 (the default) the map size is derived - /// from the available disk space on the partition that hosts the database, - /// minus `config.disk_reserve`. This allows a node to use all available - /// storage without a fixed cap. If the operator adds more storage later - /// the map is resized on demand (see [`Self::put`]). - /// - /// # Errors - /// - /// Returns an error if the LMDB environment cannot be opened. - #[allow(unsafe_code)] - pub async fn new(config: LmdbStorageConfig) -> Result { - let env_dir = config.root_dir.join("chunks.mdb"); - - // Create the directory synchronously before opening LMDB - std::fs::create_dir_all(&env_dir) - .map_err(|e| Error::Storage(format!("Failed to create LMDB directory: {e}")))?; - - let map_size = if config.max_map_size > 0 { - // Operator provided an explicit cap. - config.max_map_size - } else { - // Auto-scale: current DB footprint + available space − reserve. - let computed = compute_map_size(&env_dir, config.disk_reserve)?; - info!( - "Auto-computed LMDB map size: {:.2} GiB (data + available disk minus {:.2} GiB \ - reserve, head-room capped on Windows to bound page-table overhead)", - bytes_to_gib(computed as u64), - bytes_to_gib(config.disk_reserve), - ); - computed - }; - - let env_dir_clone = env_dir.clone(); - // Constructor-only blocking task: it runs before `self` (and its - // `blocking_tracker`) exists, so it is deliberately untracked. The - // constructor awaits it right here, so it cannot outlive this call. - let (env, db) = spawn_blocking(move || -> Result<(Env, Database)> { - // SAFETY: `EnvOpenOptions::open()` is unsafe because LMDB uses memory-mapped - // I/O and relies on OS file-locking to prevent corruption from concurrent - // access by multiple processes. We satisfy this by giving each node instance - // a unique `root_dir` (typically a directory named by its full 64-hex peer - // ID), ensuring no two processes open the same LMDB environment. Callers - // who manually configure `--root-dir` must not point multiple nodes at the - // same directory. - let env = unsafe { - EnvOpenOptions::new() - .map_size(map_size) - .max_dbs(1) - .open(&env_dir_clone) - .map_err(|e| Error::Storage(format!("Failed to open LMDB env: {e}")))? - }; - - let mut wtxn = env - .write_txn() - .map_err(|e| Error::Storage(format!("Failed to create write txn: {e}")))?; - let db: Database = env - .create_database(&mut wtxn, None) - .map_err(|e| Error::Storage(format!("Failed to create database: {e}")))?; - wtxn.commit() - .map_err(|e| Error::Storage(format!("Failed to commit db creation: {e}")))?; - - Ok((env, db)) - }) - .await - .map_err(|e| Error::Storage(format!("LMDB init task failed: {e}")))??; - - let storage = Self { - env, - db, - config, - env_dir, - stats: parking_lot::RwLock::new(StorageStats::default()), - env_lock: Arc::new(parking_lot::RwLock::new(())), - last_disk_ok: parking_lot::Mutex::new(None), - no_growth: Arc::new(AtomicBool::new(false)), - growth_pinned: Arc::new(AtomicBool::new(false)), - growth_mode_lock: tokio::sync::Mutex::new(()), - delete_growth_charged: Arc::new(AtomicU64::new(0)), - blocking_tracker: TaskTracker::new(), - #[cfg(any(test, feature = "test-utils"))] - test_put_gate: Arc::new(parking_lot::RwLock::new(())), - #[cfg(any(test, feature = "test-utils"))] - test_read_gate: Arc::new(parking_lot::RwLock::new(())), - }; - - debug!( - "Initialized LMDB storage at {:?} ({} existing chunks)", - storage.env_dir, - storage.current_chunks()? - ); - - Ok(storage) - } - - /// Store a chunk. - /// - /// Before writing, verifies that available disk space exceeds the - /// configured reserve. If the LMDB map is full but more disk space - /// exists (e.g. the operator added storage), the map is resized - /// automatically and the write is retried. - /// - /// # Returns - /// - /// Returns `true` if the chunk was newly stored, `false` if it already existed. - /// - /// # Errors - /// - /// Returns an error if the write fails, content doesn't match address, - /// or the disk is too full to accept new chunks. - pub async fn put(&self, address: &XorName, content: &[u8]) -> Result { - self.put_inner(address, content, true).await - } - - /// Store bytes under a key they do not hash to. Tests only. - /// - /// Stands in for a record that rotted in place, which is the one shape the ordinary - /// path refuses to create and the migration has to survive finding. - /// - /// # Errors - /// - /// As [`Self::put`], minus the address check. - #[cfg(test)] - pub(crate) async fn put_unchecked(&self, address: &XorName, content: &[u8]) -> Result { - self.put_inner(address, content, false).await - } - - async fn put_inner(&self, address: &XorName, content: &[u8], verify: bool) -> Result { - // Verify content address - let computed = Self::compute_address(content); - if verify && computed != *address { - return Err(Error::Storage(format!( - "Content address mismatch: expected {}, computed {}", - hex::encode(address), - hex::encode(computed) - ))); - } - - // Fast-path duplicate check (read-only, no write lock needed). - // This is an optimistic hint — the authoritative check happens inside - // the write transaction below to prevent TOCTOU races. - if self.exists(address)? { - trace!("Chunk {} already exists", hex::encode(address)); - self.stats.write().duplicates += 1; - return Ok(false); - } - - // ── Capacity guard (cached — at most one syscall per interval) ── - // Placed after the duplicate check so that re-storing an existing - // chunk remains a harmless no-op even when disk space is low. - // - // Below the reserve this pins the map instead of refusing outright, so - // the write is still attempted and LMDB decides whether a freed page - // can take it. A node that has pruned heavily keeps serving the network - // from the space it already occupies. - let no_growth = self.sync_growth_mode().await?; - - // ── Write (with resize-on-demand) ─────────────────────────────── - match self.try_put(address, content).await? { - PutOutcome::New => {} - PutOutcome::Duplicate => { - trace!("Chunk {} already exists", hex::encode(address)); - self.stats.write().duplicates += 1; - return Ok(false); - } - PutOutcome::MapFull if no_growth => { - // Both halves are now true: the volume is below the reserve and - // no free page can take *this* value. Resizing would extend the - // file into the reserve, so refuse. - // - // The refusal is not remembered. `MapFull` is specific to the - // size just attempted — a smaller value may still fit a smaller - // run — so caching it would let one maximum-sized chunk lock out - // every subsequent write. `check_capacity` estimates instead. - return Err(Error::Storage(format!( - "Insufficient disk space: {:.2} GiB reserve required and no reusable page \ - in the local store fits this {} B value. \ - Free disk space or increase the partition to continue storing chunks.", - bytes_to_gib(self.config.disk_reserve), - content.len(), - ))); - } - PutOutcome::MapFull => { - // The map ceiling was reached but there may be more disk space - // available (e.g. operator expanded the partition). - // - // Guarded: `no_growth` was sampled before the write, so the - // store may have entered no-growth mode since. Growing the map - // outside the transition lock could undo a pin that a - // concurrent `sync_growth_mode` had just applied, handing the - // reserve back to ordinary writes. - self.try_resize_for_growth().await?; - // Retry once after resize. - match self.try_put(address, content).await? { - PutOutcome::New => {} - PutOutcome::Duplicate => { - self.stats.write().duplicates += 1; - return Ok(false); - } - PutOutcome::MapFull => { - return Err(Error::Storage( - "LMDB map full after resize — disk may be at capacity".into(), - )); - } - } - } - } - - { - let mut stats = self.stats.write(); - stats.chunks_stored += 1; - stats.bytes_stored += content.len() as u64; - } - - debug!( - "Stored chunk {} ({} bytes)", - hex::encode(address), - content.len() - ); - - Ok(true) - } - - /// Attempt a single put inside a write transaction. - /// - /// Returns [`PutOutcome::MapFull`] instead of an error when the LMDB map - /// ceiling is reached, so the caller can resize and retry. - async fn try_put(&self, address: &XorName, content: &[u8]) -> Result { - let key = *address; - let value = content.to_vec(); - let env = self.env.clone(); - let db = self.db; - let lock = Arc::clone(&self.env_lock); - #[cfg(any(test, feature = "test-utils"))] - let test_put_gate = Arc::clone(&self.test_put_gate); - - self.blocking_tracker - .spawn_blocking(move || -> Result { - // Test-only: parks here while a test holds the write half. - #[cfg(any(test, feature = "test-utils"))] - let _test_put_gate = test_put_gate.read(); - let _guard = lock.read(); - - let mut wtxn = env - .write_txn() - .map_err(|e| Error::Storage(format!("Failed to create write txn: {e}")))?; - - // Authoritative existence check inside the serialized write txn - if db - .get(&wtxn, &key) - .map_err(|e| Error::Storage(format!("Failed to check existence: {e}")))? - .is_some() - { - return Ok(PutOutcome::Duplicate); - } - - match db.put(&mut wtxn, &key, &value) { - Ok(()) => {} - Err(heed::Error::Mdb(MdbError::MapFull)) => return Ok(PutOutcome::MapFull), - Err(e) => { - return Err(Error::Storage(format!("Failed to put chunk: {e}"))); - } - } - - match wtxn.commit() { - Ok(()) => Ok(PutOutcome::New), - Err(heed::Error::Mdb(MdbError::MapFull)) => Ok(PutOutcome::MapFull), - Err(e) => Err(Error::Storage(format!("Failed to commit put: {e}"))), - } - }) - .await - .map_err(|e| Error::Storage(format!("LMDB put task failed: {e}")))? - } - - /// Retrieve a chunk. - /// - /// # Returns - /// - /// Returns `Some(content)` if found, `None` if not found. - /// - /// # Errors - /// - /// Returns an error if read fails or verification fails. - pub async fn get(&self, address: &XorName) -> Result>> { - let key = *address; - let env = self.env.clone(); - let db = self.db; - let lock = Arc::clone(&self.env_lock); - - let content = self - .blocking_tracker - .spawn_blocking(move || -> Result>> { - let _guard = lock.read(); - let rtxn = env - .read_txn() - .map_err(|e| Error::Storage(format!("Failed to create read txn: {e}")))?; - let value = db - .get(&rtxn, &key) - .map_err(|e| Error::Storage(format!("Failed to get chunk: {e}")))?; - Ok(value.map(Vec::from)) - }) - .await - .map_err(|e| Error::Storage(format!("LMDB get task failed: {e}")))??; - - let Some(content) = content else { - trace!("Chunk {} not found", hex::encode(address)); - return Ok(None); - }; - - // Verify content if configured - if self.config.verify_on_read { - let computed = Self::compute_address(&content); - if computed != *address { - self.stats.write().verification_failures += 1; - warn!( - "Chunk verification failed: expected {}, computed {}", - hex::encode(address), - hex::encode(computed) - ); - return Err(Error::Storage(format!( - "Chunk verification failed for {}", - hex::encode(address) - ))); - } - } - - { - let mut stats = self.stats.write(); - stats.chunks_retrieved += 1; - stats.bytes_retrieved += content.len() as u64; - } - - debug!( - "Retrieved chunk {} ({} bytes)", - hex::encode(address), - content.len() - ); - - Ok(Some(content)) - } - - /// Check if a chunk exists. - /// - /// # Errors - /// - /// Returns an error if the LMDB read transaction fails. - pub fn exists(&self, address: &XorName) -> Result { - let _guard = self.env_lock.read(); - let rtxn = self - .env - .read_txn() - .map_err(|e| Error::Storage(format!("Failed to create read txn: {e}")))?; - let found = self - .db - .get(&rtxn, address.as_ref()) - .map_err(|e| Error::Storage(format!("Failed to check existence: {e}")))? - .is_some(); - Ok(found) - } - - /// Delete a chunk. - /// - /// # Errors - /// - /// Returns an error if deletion fails. - pub async fn delete(&self, address: &XorName) -> Result { - let key = *address; - - // Establish growth mode first, exactly as `put` does. Otherwise a - // delete arriving while the volume is low but before any write has - // pinned the map would copy-on-write into whatever head-room the - // ceiling still had, growing `data.mdb` into the reserve without - // passing through the budgeted allowance below. - self.sync_growth_mode().await?; - - let deleted = match self.try_delete(&key).await? { - DeleteOutcome::Done(existed) => existed, - DeleteOutcome::MapFull => { - // A delete is a write: LMDB copies the B-tree path before it - // frees the leaf pages, so a store with no free page at all - // cannot delete inside a map pinned to the file size. Without a - // way through, a node that filled up before it ever pruned - // could never prune its way out. - // - // Serialised against `sync_growth_mode` so the two cannot - // interleave their resizes. - let _transition = self.growth_mode_lock.lock().await; - self.delete_with_slack(&key).await? - } - }; - - if deleted { - debug!("Deleted chunk {}", hex::encode(address)); - } - - Ok(deleted) - } - - /// Attempt one delete, reporting `MapFull` rather than raising it. - async fn try_delete(&self, key: &XorName) -> Result { - let key = *key; - let env = self.env.clone(); - let db = self.db; - let lock = Arc::clone(&self.env_lock); - - self.blocking_tracker - .spawn_blocking(move || -> Result { - let _guard = lock.read(); - delete_in_txn(&env, db, &key) - }) - .await - .map_err(|e| Error::Storage(format!("LMDB delete task failed: {e}")))? - } - - /// Get storage statistics. - #[must_use] - pub fn stats(&self) -> StorageStats { - let mut stats = self.stats.read().clone(); - match self.current_chunks() { - Ok(count) => stats.current_chunks = count, - Err(e) => { - warn!("Failed to read current_chunks for stats: {e}"); - stats.current_chunks = 0; - } - } - stats - } - - /// Return the number of chunks currently stored, queried from LMDB metadata. - /// - /// This is an O(1) read of the B-tree page header — not a full scan. - /// - /// # Errors - /// - /// Returns an error if the LMDB read transaction fails. - pub fn current_chunks(&self) -> Result { - let _guard = self.env_lock.read(); - let rtxn = self - .env - .read_txn() - .map_err(|e| Error::Storage(format!("Failed to create read txn: {e}")))?; - let entries = self - .db - .stat(&rtxn) - .map_err(|e| Error::Storage(format!("Failed to read db stats: {e}")))? - .entries; - Ok(entries as u64) - } - - /// Compute content address (BLAKE3 hash). - #[must_use] - pub fn compute_address(content: &[u8]) -> XorName { - crate::client::compute_address(content) - } - - /// Get the root directory. - #[must_use] - pub fn root_dir(&self) -> &Path { - &self.config.root_dir - } - - /// Return all stored record keys. - /// - /// Iterates the LMDB database in a read transaction. Used by the - /// replication subsystem for hint construction and audit sampling. - /// - /// # Errors - /// - /// Returns an error if the LMDB read transaction fails. - pub async fn all_keys(&self) -> Result> { - let env = self.env.clone(); - let db = self.db; - let lock = Arc::clone(&self.env_lock); - - let keys = self - .blocking_tracker - .spawn_blocking(move || -> Result> { - // Hold the shared lock for the whole read so try_resize() (which - // takes the exclusive lock before the unsafe Env::resize()) cannot - // unmap the environment while this txn and its cursor are live. - let _guard = lock.read(); - let rtxn = env - .read_txn() - .map_err(|e| Error::Storage(format!("Failed to create read txn: {e}")))?; - let mut keys = Vec::new(); - let iter = db - .iter(&rtxn) - .map_err(|e| Error::Storage(format!("Failed to iterate database: {e}")))?; - for result in iter { - let (key_bytes, _) = - result.map_err(|e| Error::Storage(format!("Failed to read entry: {e}")))?; - if key_bytes.len() == XORNAME_LEN { - let mut key = [0u8; XORNAME_LEN]; - key.copy_from_slice(key_bytes); - keys.push(key); - } else { - crate::logging::warn!( - "LmdbStorage: skipping entry with unexpected key length {} (expected {XORNAME_LEN})", - key_bytes.len() - ); - } - } - Ok(keys) - }) - .await - .map_err(|e| Error::Storage(format!("all_keys task failed: {e}")))?; - - keys - } - - /// Retrieve raw chunk bytes without content-address verification. - /// - /// Used by the audit subsystem to compute digests over stored bytes. - /// Unlike [`Self::get`], this does not verify `hash(content) == address`. - /// - /// # Errors - /// - /// Returns an error if the LMDB read transaction fails. - pub async fn get_raw(&self, address: &XorName) -> Result>> { - let key = *address; - let env = self.env.clone(); - let db = self.db; - let lock = Arc::clone(&self.env_lock); - #[cfg(any(test, feature = "test-utils"))] - let test_read_gate = Arc::clone(&self.test_read_gate); - - let value = self - .blocking_tracker - .spawn_blocking(move || -> Result>> { - // Shared lock held until the bytes are copied out, so a concurrent - // try_resize() cannot unmap the environment mid-read. See all_keys. - let _guard = lock.read(); - // Test-only: parks here, still holding the shared lock, while a - // test holds the write half — used to prove a resize waits. - #[cfg(any(test, feature = "test-utils"))] - let _test_read_gate = test_read_gate.read(); - let rtxn = env - .read_txn() - .map_err(|e| Error::Storage(format!("Failed to create read txn: {e}")))?; - let val = db - .get(&rtxn, key.as_ref()) - .map_err(|e| Error::Storage(format!("Failed to get chunk: {e}")))?; - Ok(val.map(Vec::from)) - }) - .await - .map_err(|e| Error::Storage(format!("get_raw task failed: {e}")))?; - - value - } - - /// Cheap capacity pre-check for callers that want to reject work *before* - /// doing expensive setup (e.g. the PUT handler skipping payment - /// verification on a full node — see `V2-411`). - /// - /// A node is full only when **both** halves are true: the volume is below - /// the reserve *and* the store has no reusable page left. Deleting a record - /// returns its pages to LMDB's free list and never to the filesystem, so a - /// node that has pruned heavily sits on reusable capacity while `statvfs` - /// still reports the volume as full. Refusing on the disk half alone stops - /// such a node from writing into space it already owns. - /// - /// This is a **hint**, deliberately biased towards admitting: it estimates - /// reusable bytes and only refuses when there is not even one chunk's worth. - /// The authority on whether a given write fits stays with LMDB's allocator - /// in [`Self::put`], because no page count can account for the - /// copy-on-write of the B-tree path, the contiguous run a multi-megabyte - /// value needs, or pages still pinned by an open read transaction. An - /// over-optimistic hint costs one refused write; an over-pessimistic one - /// would recreate the bug this exists to fix. - /// - /// # Errors - /// - /// Returns [`Error::Storage`] when the volume is below the reserve and the - /// store holds less than one chunk of reusable space, or when the - /// disk-space query itself fails. - /// Unused while the node is moving off this store. - /// - /// Capacity is now the file store's question, because that is where writes land, and - /// this predicate deliberately answers a different one: it counts pages this store can - /// reuse internally, which says nothing about whether the *file* about to be written - /// will fit. [`Self::capacity_verdict`] is still used, to decide whether the bridge's - /// copy into this store is worth attempting. Both go when this store does. - #[allow(dead_code)] - pub(crate) fn check_capacity(&self) -> Result<()> { - let Some(available) = self.available_space_cached()? else { - return Ok(()); - }; - - let reusable = self.reusable_bytes()?; - if reusable >= MAX_CHUNK_SIZE as u64 { - return Ok(()); - } - - Err(Error::Storage(format!( - "Insufficient disk space: {:.2} GiB available, {:.2} GiB reserve required, \ - and only {reusable} B reusable inside the local store. \ - Free disk space or increase the partition to continue storing chunks.", - bytes_to_gib(available), - bytes_to_gib(self.config.disk_reserve), - ))) - } - - /// Capacity as a three-way verdict, distinguishing a full node from a - /// query that failed. - /// - /// Refuses on the same two-part predicate as [`Self::check_capacity`]: - /// `Full` only when the volume is below the reserve *and* the store holds - /// less than one chunk of reusable space. Deleted records return their - /// pages to LMDB's free list and never to the filesystem, so a pruned node - /// reads as full to `statvfs` while still able to store chunks — such a - /// node must keep discovering holders for the keys it owes. - /// - /// Shares the same TTL cache as [`Self::check_capacity`]: a passing disk - /// reading is cached, a failing one is always rechecked, so freed space is - /// noticed promptly. An admit that rests on reusable pages is deliberately - /// not cached, matching the pre-check, because the free list can drain a - /// chunk at a time. - pub(crate) fn capacity_verdict(&self) -> CapacityVerdict { - { - let last = self.last_disk_ok.lock(); - if let Some(t) = *last { - if t.elapsed().as_secs() < DISK_CHECK_INTERVAL_SECS { - return CapacityVerdict::Writable; - } - } - } - let disk = verdict_from_available_space( - fs2::available_space(&self.env_dir), - self.config.disk_reserve, - ); - match disk { - CapacityVerdict::Writable => { - *self.last_disk_ok.lock() = Some(Instant::now()); - CapacityVerdict::Writable - } - CapacityVerdict::Unknown => CapacityVerdict::Unknown, - // Below the reserve is only half the predicate: the store may - // still hold pages it can reuse without growing the file. - CapacityVerdict::Full => match self.reusable_bytes() { - Ok(reusable) if reusable >= MAX_CHUNK_SIZE as u64 => CapacityVerdict::Writable, - Ok(_) => CapacityVerdict::Full, - Err(e) => { - warn!("Could not query the store's reusable space: {e}"); - CapacityVerdict::Unknown - } - }, - } - } - - /// Estimated bytes inside `data.mdb` that LMDB could write without growing - /// the file: the file size minus the pages currently holding data. - /// - /// Deliberately an over-estimate. `stat()` counts only the branch, leaf and - /// overflow pages of the unnamed database, so the free-list's own pages and - /// the environment metadata fall on the "reusable" side. Erring high keeps - /// [`Self::check_capacity`] biased towards admitting the attempt. - /// - /// Uses `stat()` rather than heed's `non_free_pages_size()`, which walks the - /// unnamed database calling `String::from_utf8(key).unwrap()` on every key - /// without a zero byte. Our keys are 32 random bytes, so that call panics - /// almost immediately. A single unnamed database makes `stat()` equivalent. - fn reusable_bytes(&self) -> Result { - // Order matters. The two samples are not atomic, so read the live pages - // first and the file length second: a write committing in between then - // pairs an older (smaller) live count with a newer (larger) file, which - // over-estimates. Sampling the other way round pairs a stale file - // length with a fresh live count and can under-estimate, which would - // refuse a node that has room — the very bug this fixes. - let stat = self.env.stat(); - let live_pages = (stat.branch_pages as u64) - .saturating_add(stat.leaf_pages as u64) - .saturating_add(stat.overflow_pages as u64); - let live_bytes = live_pages.saturating_mul(u64::from(stat.page_size)); - - let file_bytes = self - .env - .real_disk_size() - .map_err(|e| Error::Storage(format!("Failed to query LMDB file size: {e}")))?; - - Ok(file_bytes.saturating_sub(live_bytes)) - } - - /// Available bytes on the storage volume, or `None` when a recent check - /// already showed it above the reserve. - /// - /// Only *passing* results are cached, so a low-space condition is always - /// re-measured and freed space is detected promptly. - fn available_space_cached(&self) -> Result> { - { - let last = self.last_disk_ok.lock(); - if let Some(t) = *last { - if t.elapsed().as_secs() < DISK_CHECK_INTERVAL_SECS { - return Ok(None); - } - } - } - - let available = fs2::available_space(&self.env_dir) - .map_err(|e| Error::Storage(format!("Failed to query available disk space: {e}")))?; - - if available >= self.config.disk_reserve { - *self.last_disk_ok.lock() = Some(Instant::now()); - return Ok(None); - } - - Ok(Some(available)) - } - - /// Align the map ceiling with the current disk state, returning whether the - /// store is in no-growth mode. - /// - /// Below the reserve the map is pinned to the file's high-water mark, so a - /// put succeeds exactly when LMDB can satisfy it from the free list and - /// returns `MapFull` the moment it would need to extend `data.mdb`. That - /// makes the allocator the authority on "can this write fit". - /// - /// The whole transition runs under `growth_mode_lock`. Setting the flag and - /// resizing the map is one compound change spanning an await, so without - /// serialisation two callers straddling the threshold can interleave and - /// leave the flag describing a map that was never applied. - async fn sync_growth_mode(&self) -> Result { - let _transition = self.growth_mode_lock.lock().await; - - // Re-measured inside the lock: a caller that queued behind a transition - // must act on the state that transition left behind, not the one it saw - // before waiting. - // Pinned for the bridge: never unpinned by having room, because the room is not - // this environment's to spend while another store is measuring the same disk. - if self.growth_pinned.load(Ordering::Acquire) { - self.no_growth.store(true, Ordering::Release); - self.pin_map_to_high_water().await?; - return Ok(true); - } - if self.available_space_cached()?.is_none() { - // At or above the reserve: restore normal head-room if we pinned it. - if self.no_growth.load(Ordering::Acquire) { - // Intent first, work second. A `spawn_blocking` body outlives a - // cancelled awaiter, so ordering between two resizes cannot be - // guaranteed by holding an async lock. Publishing the intent - // before the work lets each closure re-read it under the - // exclusive lock and decline if it has since been reversed. - self.no_growth.store(false, Ordering::Release); - self.try_resize().await?; - } - // Real disk again: the maintenance allowance is refreshed. Done on - // every healthy pass, not just the transition, so an allowance - // spent while the flag happened to be clear is still returned. - self.delete_growth_charged.store(0, Ordering::Release); - return Ok(false); - } - - // Called unconditionally, not just on the transition. A re-pin that - // failed, or a transition whose caller was cancelled while its detached - // resize was still in flight, can leave the flag set while the map is - // not actually pinned; re-asserting it here repairs that instead of - // trusting the flag. The call is a no-op when already pinned. - self.no_growth.store(true, Ordering::Release); - self.pin_map_to_high_water().await?; - - Ok(true) - } - - /// Keep this environment from ever claiming new disk, until the process ends. - /// - /// For the migration bridge, where a second store measures the same free space and - /// neither knows what the other is about to spend. Pinned, this one writes only from - /// pages it already holds, so the other's accounting is the only claim on free disk. - /// - /// # Errors - /// - /// Returns [`Error::Storage`] if the map cannot be pinned. - pub async fn pin_growth(&self) -> Result<()> { - self.growth_pinned.store(true, Ordering::Release); - self.sync_growth_mode().await.map(|_| ()) - } - - /// Pin the LMDB map to the size of `data.mdb` on disk. - /// - /// Every page already in the file stays usable, including free ones, but - /// the file cannot grow, so the configured reserve is preserved by LMDB - /// itself rather than by refusing writes it could have served. - /// - /// Deliberately leaves **no** head-room: any slack in the ceiling is - /// ordinary put capacity, so it would be spent on the next chunk rather - /// than kept for maintenance, and on a shared volume every node would take - /// its own slice out of the reserve. Deletes get their copy-on-write room - /// on demand instead, see [`Self::delete`]. - /// - /// Takes the **exclusive** `env_lock` for the same reason - /// [`Self::try_resize`] does: `mdb_env_set_mapsize` requires that no - /// transaction is active. Callers hold `growth_mode_lock`. - #[allow(unsafe_code)] - async fn pin_map_to_high_water(&self) -> Result<()> { - // The "is it already pinned?" test lives inside the exclusive lock - // below, not out here. An unlocked pre-check can observe "already - // pinned" moments before a detached resize from a cancelled transition - // lands, after which the flag would claim a pin that no longer holds. - // Callers invoke this on every low-disk write so the pinned state - // repairs itself; the locked section is a few reads when nothing is to - // be done. - let env = self.env.clone(); - let lock = Arc::clone(&self.env_lock); - let no_growth = Arc::clone(&self.no_growth); - - self.blocking_tracker - .spawn_blocking(move || -> Result<()> { - // Exclusive lock guarantees no concurrent transactions. - let _guard = lock.write(); - - // Re-read under the lock: this closure may have been queued - // behind others, or its awaiter cancelled, and the store may - // have left no-growth mode since it was spawned. - if !no_growth.load(Ordering::Acquire) { - return Ok(()); - } - - let current_map = env.info().map_size; - let file_bytes = env - .real_disk_size() - .map_err(|e| Error::Storage(format!("Failed to query LMDB file size: {e}")))?; - - let page = page_size::get() as u64; - let aligned = file_bytes.div_ceil(page) * page; - let target = usize::try_from(aligned).unwrap_or(usize::MAX); - - // Re-checked under the lock: the state may have moved between - // the cheap check and here. - if target >= current_map { - return Ok(()); - } - - // SAFETY: We hold an exclusive lock, so no transactions are active. - unsafe { - env.resize(target) - .map_err(|e| Error::Storage(format!("Failed to pin LMDB map: {e}")))?; - } - - info!( - "Disk below reserve: pinned LMDB map to {:.2} GiB (was {:.2} GiB). \ - Writes that fit in already-freed pages still succeed; \ - only writes that would grow the file are refused.", - bytes_to_gib(target as u64), - bytes_to_gib(current_map as u64), - ); - Ok(()) - }) - .await - .map_err(|e| Error::Storage(format!("LMDB map pin task failed: {e}")))? - } - - /// Grow the map for a write, unless the store is pinned below the reserve. - /// - /// Serialised against [`Self::sync_growth_mode`] so a resize cannot land - /// after a pin and quietly undo it. If the store entered no-growth mode - /// while the write was in flight, the caller's `MapFull` is final and no - /// growth happens. - async fn try_resize_for_growth(&self) -> Result<()> { - let _transition = self.growth_mode_lock.lock().await; - - if self.no_growth.load(Ordering::Acquire) { - return Ok(()); - } - - self.try_resize().await - } - - /// Delete `key` with [`DELETE_COW_SLACK`] of temporary map head-room, then - /// take the head-room straight back. - /// - /// The raise, the delete and the re-pin all happen inside **one** exclusive - /// `env_lock` scope. Doing them as three separate locked steps would leave - /// windows in which an ordinary put could allocate from the raised ceiling, - /// spending the reserve on a chunk instead of on the maintenance it was - /// granted for, and an error or cancellation between the steps would leave - /// the ceiling raised for good. - /// - /// What is budgeted is the *growth*, not the grant. If the copy-on-write - /// does extend `data.mdb` that growth is permanent, since LMDB never - /// returns file space, so repeated fill-then-delete cycles could otherwise - /// walk the file into the reserve a slice at a time. A delete that finds - /// room inside the file is charged nothing. - /// - /// Charging per grant instead would be wrong, and was: page reuse is not - /// reliably available to the very next delete, because LMDB will not hand - /// back pages a still-recent transaction freed. A one-grant budget - /// therefore stopped a node pruning after its first assisted delete, which - /// showed up as every delete failing on 4 KiB-page hosts while passing on - /// 16 KiB-page ones. The budget resets when the store leaves no-growth - /// mode, i.e. when there is real disk to work with again. - #[allow(unsafe_code)] - async fn delete_with_slack(&self, key: &XorName) -> Result { - let key = *key; - let env = self.env.clone(); - let db = self.db; - let lock = Arc::clone(&self.env_lock); - let budget = Arc::clone(&self.delete_growth_charged); - - let outcome = self - .blocking_tracker - .spawn_blocking(move || -> Result { - // Checked and charged entirely inside the closure. A - // `spawn_blocking` body keeps running when its awaiter is - // dropped, so accounting split across the await could be - // skipped, permanently costing the node its ability to prune. - if budget.load(Ordering::Acquire) >= DELETE_COW_GROWTH_BUDGET { - return Err(Error::Storage(format!( - "Cannot delete: the local store is full and deletes have already used \ - their {DELETE_COW_GROWTH_BUDGET} B growth allowance. \ - Free disk space to continue." - ))); - } - - // Exclusive for the whole sequence: no transaction may be - // active across either resize, and no put may observe the - // raised ceiling. - let _guard = lock.write(); - - let page = page_size::get() as u64; - let previous_map = env.info().map_size; - let file_before = env - .real_disk_size() - .map_err(|e| Error::Storage(format!("Failed to query LMDB file size: {e}")))?; - let raised = (previous_map as u64) - .saturating_add(DELETE_COW_SLACK) - .div_ceil(page) - .saturating_mul(page); - - // SAFETY: exclusive lock held, so no transactions are active. - let granted = unsafe { - env.resize(usize::try_from(raised).unwrap_or(usize::MAX)) - .map_err(|e| Error::Storage(format!("Failed to grant delete slack: {e}"))) - }; - granted?; - - // Armed across the delete so an unwind still restores the - // ceiling; disarmed once the explicit restore below succeeds. - let mut ceiling_guard = MapCeilingRestorer { - env: &env, - previous: previous_map, - armed: true, - }; - - let outcome = delete_in_txn(&env, db, &key); - - // Charge what the file actually gained, not the fact that slack - // was offered. A delete that found room inside `data.mdb` costs - // nothing and must not consume the allowance, otherwise a node - // stops being able to prune after its first assisted delete. - // Measured before the ceiling is restored, and before any error - // is propagated, so a committed delete is always accounted for. - // - // A measurement that fails is charged the whole slack rather than nothing. - // Reading it back as the size before the delete would say the file did not - // grow, and a delete that did grow would then spend disk the budget never - // saw. Repeat that and the ceiling stops meaning anything. Over-charging - // costs at worst one assisted delete; under-charging costs the reserve. - let grew = match env.real_disk_size() { - Ok(file_after) => file_after.saturating_sub(file_before), - Err(e) => { - warn!( - "Could not measure the LMDB file after an assisted delete \ - ({e}); charging the whole slack rather than assuming it cost \ - nothing" - ); - DELETE_COW_SLACK - } - }; - if grew > 0 { - budget.fetch_add(grew, Ordering::AcqRel); - } - - // Undo the raise before releasing the lock, on every path and - // whatever the delete did. Restoring to the previous ceiling - // rather than to a freshly measured file size keeps this - // unconditional: it is exactly the inverse of the raise, needs - // no second syscall that could itself fail, and is correct - // whether or not the store was pinned. If the copy-on-write did - // extend the file, LMDB clamps a request below the space in use, - // so the map still covers the data. - // - // SAFETY: exclusive lock held, so no transactions are active. - let restored = unsafe { - env.resize(previous_map) - .map_err(|e| Error::Storage(format!("Failed to restore LMDB map: {e}"))) - }; - if restored.is_ok() { - ceiling_guard.armed = false; - } - - // A failed restore is reported ahead of a failed delete, so the - // failure is not lost behind the delete's own error. - match (outcome, restored) { - (Ok(outcome), Ok(())) => Ok(outcome), - (_, Err(e)) | (Err(e), Ok(())) => Err(e), - } - }) - .await - .map_err(|e| Error::Storage(format!("LMDB delete-slack task failed: {e}")))?; - - match outcome? { - DeleteOutcome::Done(existed) => Ok(existed), - DeleteOutcome::MapFull => Err(Error::Storage( - "LMDB map full during delete even with the maintenance allowance".into(), - )), - } - } - - /// Grow the LMDB map to match currently available disk space. - /// - /// The new size is the **larger** of: - /// 1. the current map size (so existing data is never truncated), and - /// 2. `current_db_file_size + available_space − reserve` - /// (so all reachable disk space can be used). - /// - /// Acquires an **exclusive** lock on `env_lock` so that no read or write - /// transactions are active when the underlying `mdb_env_set_mapsize` is - /// called (an LMDB safety requirement). - #[allow(unsafe_code)] - async fn try_resize(&self) -> Result<()> { - let env = self.env.clone(); - let lock = Arc::clone(&self.env_lock); - let no_growth = Arc::clone(&self.no_growth); - let env_dir = self.env_dir.clone(); - let reserve = self.config.disk_reserve; - - self.blocking_tracker - .spawn_blocking(move || -> Result<()> { - // Exclusive lock guarantees no concurrent transactions. - let _guard = lock.write(); - - // Re-read under the lock. A `spawn_blocking` body outlives a - // cancelled awaiter, so this closure may land after the store - // entered no-growth mode. Growing then would hand back the - // head-room a pin had just taken away, and with it the disk - // reserve. - if no_growth.load(Ordering::Acquire) { - return Ok(()); - } - - // Measured here rather than before the spawn, so a late closure - // sizes from the disk as it is now, not as it was when queued. - let from_disk = compute_map_size(&env_dir, reserve)?; - - // Never shrink below the current map — existing data must remain - // addressable regardless of what the disk-space calculation says. - let current_map = env.info().map_size; - let new_size = from_disk.max(current_map); - - if new_size <= current_map { - debug!("LMDB map resize skipped — no additional disk space available"); - return Ok(()); - } - - // SAFETY: We hold an exclusive lock, so no transactions are active. - unsafe { - env.resize(new_size) - .map_err(|e| Error::Storage(format!("Failed to resize LMDB map: {e}")))?; - } - - info!( - "Resized LMDB map to {:.2} GiB (was {:.2} GiB)", - bytes_to_gib(new_size as u64), - bytes_to_gib(current_map as u64), - ); - Ok(()) - }) - .await - .map_err(|e| Error::Storage(format!("LMDB resize task failed: {e}")))? - } - - /// Wait until every tracked LMDB blocking task has finished. - /// - /// Dropping an async caller (e.g. a `select!` losing to a shutdown token) - /// does not cancel an already-spawned blocking closure — the closure keeps - /// running on the blocking pool with a cloned [`Env`]. This method waits - /// for those detached closures too, so when it returns no blocking - /// operation still holds the environment. - /// - /// Quiescence is only meaningful once callers have stopped issuing new - /// operations; concurrent traffic can keep the tracker non-empty - /// indefinitely. The storage remains fully usable afterwards (the - /// internal tracker is reopened before returning). - pub async fn wait_idle(&self) { - self.blocking_tracker.close(); - self.blocking_tracker.wait().await; - self.blocking_tracker.reopen(); - } - - /// Test-only handle to the put gate. - /// - /// Hold the write half to deterministically park the next put inside its - /// blocking closure (e.g. to exercise [`Self::wait_idle`] with a write - /// still in flight). - #[cfg(any(test, feature = "test-utils"))] - #[must_use] - pub fn test_put_gate(&self) -> Arc> { - Arc::clone(&self.test_put_gate) - } - - /// Test-only handle to the raw-read gate. - /// - /// Hold the write half to deterministically park the next raw read - /// (`get_raw`) inside its blocking closure while it still holds the shared - /// environment lock (e.g. to prove `try_resize` waits for active - /// raw reads). - #[cfg(any(test, feature = "test-utils"))] - #[must_use] - pub fn test_read_gate(&self) -> Arc> { - Arc::clone(&self.test_read_gate) - } -} - -// ──────────────────────────────────────────────────────────────────────────── -// Helpers -// ──────────────────────────────────────────────────────────────────────────── - -/// Outcome of a single `try_put` attempt. -enum PutOutcome { - /// Chunk was newly stored. - New, - /// Chunk already existed (idempotent). - Duplicate, - /// The LMDB map ceiling was reached — caller should resize and retry. - MapFull, -} - -/// Restores an LMDB map ceiling when dropped, including while unwinding. -/// -/// The explicit restore in [`LmdbStorage::delete_with_slack`] is the normal -/// path, because it can report a failure to the caller. This exists so a panic -/// between the raise and that restore cannot leave the ceiling raised, which -/// would quietly hand ordinary writes the disk reserve. -struct MapCeilingRestorer<'a> { - env: &'a Env, - previous: usize, - armed: bool, -} - -impl Drop for MapCeilingRestorer<'_> { - #[allow(unsafe_code)] - fn drop(&mut self) { - if !self.armed { - return; - } - // SAFETY: the owner holds the exclusive `env_lock` for this whole - // scope, so no transaction is active. - unsafe { - if let Err(e) = self.env.resize(self.previous) { - warn!("Failed to restore the LMDB map ceiling while unwinding: {e}"); - } - } - } -} - -/// Run one delete in its own write transaction, reporting `MapFull` rather than -/// raising it. -/// -/// The caller owns the `env_lock` discipline: [`LmdbStorage::try_delete`] holds -/// the shared guard, [`LmdbStorage::delete_with_slack`] the exclusive one. -fn delete_in_txn(env: &Env, db: Database, key: &XorName) -> Result { - let mut wtxn = match env.write_txn() { - Ok(wtxn) => wtxn, - Err(heed::Error::Mdb(MdbError::MapFull)) => return Ok(DeleteOutcome::MapFull), - Err(e) => return Err(Error::Storage(format!("Failed to create write txn: {e}"))), - }; - let existed = match db.delete(&mut wtxn, key) { - Ok(existed) => existed, - Err(heed::Error::Mdb(MdbError::MapFull)) => return Ok(DeleteOutcome::MapFull), - Err(e) => return Err(Error::Storage(format!("Failed to delete chunk: {e}"))), - }; - match wtxn.commit() { - Ok(()) => Ok(DeleteOutcome::Done(existed)), - Err(heed::Error::Mdb(MdbError::MapFull)) => Ok(DeleteOutcome::MapFull), - Err(e) => Err(Error::Storage(format!("Failed to commit delete: {e}"))), - } -} - -/// Outcome of one delete attempt. -enum DeleteOutcome { - /// The delete committed; the flag is whether the key had existed. - Done(bool), - /// The map ceiling left no room for the delete's copy-on-write. - MapFull, -} - -/// Compute the LMDB map size from the disk hosting `db_dir`. -/// -/// The result covers **all existing data** plus all remaining usable disk -/// space: -/// -/// ```text -/// map_size = current_db_file_size + max(0, available_space − reserve) -/// ``` -/// -/// `available_space` (from `statvfs`) reports only the *free* bytes on the -/// partition — the DB file's own footprint is **not** included, so adding -/// it back ensures the map is always large enough for the data already -/// stored. -/// -/// On Windows the disk-headroom term is additionally capped at -/// `WINDOWS_MAP_HEADROOM` to bound the map-proportional commit overhead (see -/// that constant); [`LmdbStorage::try_resize`] extends the map on demand as -/// data grows. -/// -/// The result is page-aligned and never falls below [`MIN_MAP_SIZE`]. -fn compute_map_size(db_dir: &Path, reserve: u64) -> Result { - let available = fs2::available_space(db_dir) - .map_err(|e| Error::Storage(format!("Failed to query available disk space: {e}")))?; - - // The MDB data file may not exist yet on first run, and that is the only reason to - // read zero here. Any other failure is a question that was not answered, and answering - // it with zero sizes the map as though the database were empty, which on a node with a - // large one is a map far too small to open it. - let mdb_file = db_dir.join("data.mdb"); - let current_db_bytes = match std::fs::metadata(&mdb_file) { - Ok(meta) => meta.len(), - Err(e) if e.kind() == std::io::ErrorKind::NotFound => 0, - Err(e) => { - return Err(Error::Storage(format!( - "Failed to measure {}: {e}. Refusing to size the map as though it were \ - empty.", - mdb_file.display() - ))) - } - }; - - let target = map_target_bytes(current_db_bytes, available, reserve); - - // Align up to system page size (required by heed's resize). - let page = page_size::get() as u64; - let aligned = target.div_ceil(page) * page; - - let result = usize::try_from(aligned).unwrap_or(usize::MAX); - Ok(result.max(MIN_MAP_SIZE)) -} - -/// Head-room policy for the LMDB map, split out from [`compute_map_size`] so it -/// is unit-testable without touching the real filesystem. -/// -/// `map = current_db_bytes + max(0, available − reserve)`, with the head-room -/// term capped at `WINDOWS_MAP_HEADROOM` on Windows. Existing data -/// (`current_db_bytes`) is always covered so a resize can never truncate the -/// database, even when the head-room cap or a nearly-full disk drives the -/// growth term to zero. -fn map_target_bytes(current_db_bytes: u64, available: u64, reserve: u64) -> u64 { - // available_space excludes the DB file, so we add it back to get the - // total space the DB could occupy while still leaving `reserve` free. - let growth_room = available.saturating_sub(reserve); - - // On Windows, bound the head-room so the mapped size (and its - // size-proportional commit overhead) stays proportional to stored data - // rather than disk capacity. Elsewhere, use all reachable space. - #[cfg(windows)] - let growth_room = growth_room.min(WINDOWS_MAP_HEADROOM); - - current_db_bytes.saturating_add(growth_room) -} - -/// Map the result of a space query onto the *disk half* of the capacity -/// verdict. -/// -/// `Full` here means "below the reserve", which since ant-node #210 is only -/// half the refusal predicate: [`LmdbStorage::capacity_verdict`] goes on to -/// consult the store's reusable pages before refusing. -/// -/// Split out from [`LmdbStorage::capacity_verdict`] because the three-way -/// mapping is the part worth proving, and proving it through the filesystem is -/// not portable: asking for the free space of a directory that does not exist -/// fails on Unix but succeeds on Windows, which resolves it to the volume. -fn verdict_from_available_space(available: std::io::Result, reserve: u64) -> CapacityVerdict { - match available { - Ok(available) if available < reserve => CapacityVerdict::Full, - Ok(_) => CapacityVerdict::Writable, - Err(e) => { - warn!("Could not query available disk space: {e}"); - CapacityVerdict::Unknown - } - } -} - -/// What a capacity check concluded, when the caller needs to tell "this node is -/// full" apart from "this node could not find out". -/// -/// [`LmdbStorage::check_capacity`] collapses both into `Err`, which is right for -/// a caller that only wants to know whether to attempt a write. A caller -/// deciding *how long to stand down* needs the distinction: a full disk is a -/// standing condition worth waiting minutes on, while a failed `statvfs` may -/// have cleared by the next attempt and must not be treated as one. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum CapacityVerdict { - /// Available space is at or above the configured reserve. That is what the - /// query establishes, and possibly from the TTL cache — not a promise the - /// next write succeeds. - Writable, - /// Available space is below the configured reserve. - Full, - /// The query itself failed, so nothing is known about available space. - Unknown, -} - -#[cfg(test)] -#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] -mod tests { - use super::*; - use crate::ant_protocol::MAX_CHUNK_SIZE; - - /// Short probe used to prove `wait_idle` is still blocked on a parked op. - const WAIT_IDLE_BLOCKED_PROBE: std::time::Duration = std::time::Duration::from_millis(200); - /// Generous ceiling for `wait_idle` to complete once the op is released. - const WAIT_IDLE_COMPLETE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); - /// Poll interval while waiting for a parked raw read to take the shared lock. - const RAW_READ_LOCK_POLL: std::time::Duration = std::time::Duration::from_millis(5); - /// Ceiling for a parked raw read to take the shared lock. - const RAW_READ_LOCK_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); - /// Short probe used to prove `try_resize` is still blocked on the shared lock. - const RESIZE_BLOCKED_PROBE: std::time::Duration = std::time::Duration::from_millis(200); - /// Generous ceiling for `try_resize` to complete once the raw read releases. - const RESIZE_COMPLETE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); - - #[test] - fn map_target_covers_existing_data_and_headroom() { - // A partition with far more free space than any node needs. - let huge_free = 100 * 1024 * GIB; // 100 TiB - let reserve = 500 * MIB; - - // Fresh node, no data yet. - let fresh = map_target_bytes(0, huge_free, reserve); - // Node already holding 16 GiB of chunks. - let with_data = map_target_bytes(16 * GIB, huge_free, reserve); - - #[cfg(windows)] - { - // Windows: head-room is capped, so the map (and thus page tables) - // stay bounded regardless of disk size. Existing data always sits - // on top of the capped head-room. - assert_eq!(fresh, WINDOWS_MAP_HEADROOM); - assert_eq!(with_data, 16 * GIB + WINDOWS_MAP_HEADROOM); - // Sanity: page-table cost (~map/512) is tens of MiB, not tens of GiB. - assert!(with_data / 512 < 128 * MIB); - } - - #[cfg(not(windows))] - { - // Other platforms keep the disk-sized map (lazy page tables cost - // nothing), so head-room is the full free span minus reserve. - assert_eq!(fresh, huge_free - reserve); - assert_eq!(with_data, 16 * GIB + (huge_free - reserve)); - } - } - - #[test] - fn map_target_never_truncates_data_when_disk_nearly_full() { - // Free space below the reserve → head-room saturates to 0 on every - // platform, but the existing 4 GiB of data must still be covered. - assert_eq!(map_target_bytes(4 * GIB, 100 * MIB, 500 * MIB), 4 * GIB); - } - - /// Regression (V2-620 review): `all_keys` and `get_raw` must take the shared - /// `env_lock` so their LMDB read transaction can never run concurrently with - /// `try_resize()`'s unsafe `Env::resize()` — which this PR turns into a - /// routine ~per-32-GiB Windows event. We prove it by holding the exclusive - /// lock (as a resize does) and asserting both calls block until it is freed. - /// - /// Holding a `parking_lot` guard across `.await` is deliberate and safe here: - /// the guard stays on this current-thread test task while `all_keys`/`get_raw` - /// run their blocking work on the `spawn_blocking` pool (separate threads). - #[tokio::test] - #[allow(clippy::await_holding_lock)] - async fn read_paths_block_while_env_is_being_resized() { - use std::time::Duration; - let (storage, _temp) = create_test_storage().await; - - // Store one chunk so the read paths have real work to return. - let content = b"resize-safety"; - let address = LmdbStorage::compute_address(content); - storage.put(&address, content).await.expect("put"); - - // Simulate a resize in progress: hold the exclusive env lock. - let write_guard = storage.env_lock.write(); - - // Neither read path may complete while the exclusive lock is held — - // before the fix they took no lock and would return immediately. - assert!( - tokio::time::timeout(Duration::from_millis(250), storage.all_keys()) - .await - .is_err(), - "all_keys completed while env_lock was held exclusively — missing shared guard" - ); - assert!( - tokio::time::timeout(Duration::from_millis(250), storage.get_raw(&address)) - .await - .is_err(), - "get_raw completed while env_lock was held exclusively — missing shared guard" - ); - - // Once the exclusive lock is released, both proceed and see the data. - drop(write_guard); - assert_eq!(storage.all_keys().await.expect("all_keys").len(), 1); - assert_eq!( - storage.get_raw(&address).await.expect("get_raw").as_deref(), - Some(content.as_slice()) - ); - } - - async fn create_test_storage() -> (LmdbStorage, tempfile::TempDir) { - let temp_dir = tempfile::TempDir::new().expect("create temp dir"); - let config = LmdbStorageConfig { - root_dir: temp_dir.path().to_path_buf(), - ..LmdbStorageConfig::test_default() - }; - let storage = LmdbStorage::new(config).await.expect("create storage"); - (storage, temp_dir) - } - - #[tokio::test] - async fn test_put_and_get() { - let (storage, _temp) = create_test_storage().await; - - let content = b"hello world"; - let address = LmdbStorage::compute_address(content); - - // Store chunk - let is_new = storage.put(&address, content).await.expect("put"); - assert!(is_new); - - // Retrieve chunk - let retrieved = storage.get(&address).await.expect("get"); - assert_eq!(retrieved, Some(content.to_vec())); - } - - #[tokio::test] - async fn test_put_duplicate() { - let (storage, _temp) = create_test_storage().await; - - let content = b"test data"; - let address = LmdbStorage::compute_address(content); - - // First store - let is_new1 = storage.put(&address, content).await.expect("put 1"); - assert!(is_new1); - - // Duplicate store - let is_new2 = storage.put(&address, content).await.expect("put 2"); - assert!(!is_new2); - - // Check stats - let stats = storage.stats(); - assert_eq!(stats.chunks_stored, 1); - assert_eq!(stats.duplicates, 1); - } - - #[tokio::test] - async fn test_get_not_found() { - let (storage, _temp) = create_test_storage().await; - - let address = [0xAB; 32]; - let result = storage.get(&address).await.expect("get"); - assert!(result.is_none()); - } - - #[tokio::test] - async fn test_exists() { - let (storage, _temp) = create_test_storage().await; - - let content = b"exists test"; - let address = LmdbStorage::compute_address(content); - - assert!(!storage.exists(&address).expect("exists")); - - storage.put(&address, content).await.expect("put"); - - assert!(storage.exists(&address).expect("exists")); - } - - #[tokio::test] - async fn test_delete() { - let (storage, _temp) = create_test_storage().await; - - let content = b"delete test"; - let address = LmdbStorage::compute_address(content); - - // Store - storage.put(&address, content).await.expect("put"); - assert!(storage.exists(&address).expect("exists")); - - // Delete - let deleted = storage.delete(&address).await.expect("delete"); - assert!(deleted); - assert!(!storage.exists(&address).expect("exists")); - - // Delete again (already deleted) - let deleted2 = storage.delete(&address).await.expect("delete 2"); - assert!(!deleted2); - } - - #[tokio::test] - async fn test_address_mismatch() { - let (storage, _temp) = create_test_storage().await; - - let content = b"some content"; - let wrong_address = [0xFF; 32]; // Wrong address - - let result = storage.put(&wrong_address, content).await; - assert!(result.is_err()); - assert!(result.unwrap_err().to_string().contains("mismatch")); - } - - #[test] - fn test_compute_address() { - // Known BLAKE3 hash of "hello world" - let content = b"hello world"; - let address = LmdbStorage::compute_address(content); - - let expected_hex = "d74981efa70a0c880b8d8c1985d075dbcbf679b99a5f9914e5aaf96b831a9e24"; - assert_eq!(hex::encode(address), expected_hex); - } - - #[tokio::test] - async fn test_stats() { - let (storage, _temp) = create_test_storage().await; - - let content1 = b"content 1"; - let content2 = b"content 2"; - let address1 = LmdbStorage::compute_address(content1); - let address2 = LmdbStorage::compute_address(content2); - - // Store two chunks - storage.put(&address1, content1).await.expect("put 1"); - storage.put(&address2, content2).await.expect("put 2"); - - // Retrieve one - storage.get(&address1).await.expect("get"); - - let stats = storage.stats(); - assert_eq!(stats.chunks_stored, 2); - assert_eq!(stats.chunks_retrieved, 1); - assert_eq!( - stats.bytes_stored, - content1.len() as u64 + content2.len() as u64 - ); - assert_eq!(stats.bytes_retrieved, content1.len() as u64); - assert_eq!(stats.current_chunks, 2); - } - - #[tokio::test] - async fn test_persistence_across_reopen() { - let temp_dir = tempfile::TempDir::new().expect("create temp dir"); - let content = b"persistent data"; - let address = LmdbStorage::compute_address(content); - - // Store a chunk - { - let config = LmdbStorageConfig { - root_dir: temp_dir.path().to_path_buf(), - ..LmdbStorageConfig::test_default() - }; - let storage = LmdbStorage::new(config).await.expect("create storage"); - storage.put(&address, content).await.expect("put"); - } - - // Re-open and verify it persisted - { - let config = LmdbStorageConfig { - root_dir: temp_dir.path().to_path_buf(), - ..LmdbStorageConfig::test_default() - }; - let storage = LmdbStorage::new(config).await.expect("reopen storage"); - assert_eq!(storage.current_chunks().expect("current_chunks"), 1); - let retrieved = storage.get(&address).await.expect("get"); - assert_eq!(retrieved, Some(content.to_vec())); - } - } - - #[tokio::test] - async fn test_all_keys() { - let (storage, _temp) = create_test_storage().await; - - // Empty storage - let keys = storage.all_keys().await.expect("all_keys empty"); - assert!(keys.is_empty()); - - // Store some chunks - let content1 = b"chunk one for keys"; - let content2 = b"chunk two for keys"; - let addr1 = LmdbStorage::compute_address(content1); - let addr2 = LmdbStorage::compute_address(content2); - storage.put(&addr1, content1).await.expect("put 1"); - storage.put(&addr2, content2).await.expect("put 2"); - - let mut keys = storage.all_keys().await.expect("all_keys"); - keys.sort_unstable(); - let mut expected = vec![addr1, addr2]; - expected.sort_unstable(); - assert_eq!(keys, expected); - } - - #[tokio::test] - async fn test_get_raw() { - let (storage, _temp) = create_test_storage().await; - - let content = b"raw test data"; - let address = LmdbStorage::compute_address(content); - storage.put(&address, content).await.expect("put"); - - // get_raw returns bytes without verification - let raw = storage.get_raw(&address).await.expect("get_raw"); - assert_eq!(raw, Some(content.to_vec())); - - // Non-existent key - let missing = storage.get_raw(&[0xFF; 32]).await.expect("get_raw missing"); - assert!(missing.is_none()); - } - - /// Dropping a put's awaiter does not cancel its `spawn_blocking` LMDB - /// transaction; `wait_idle` must wait for that detached write, and the - /// storage must remain usable afterwards. - // Holding the gate's write guard across awaits is the point of the test: - // it parks the blocking closure while we probe wait_idle. - #[allow(clippy::await_holding_lock)] - #[tokio::test] - async fn wait_idle_waits_for_detached_put_blocking_op() { - let (storage, _temp) = create_test_storage().await; - - let content = b"detached put survives its dropped awaiter"; - let address = LmdbStorage::compute_address(content); - - // Park the put's blocking closure on the test gate. - let gate = storage.test_put_gate(); - let parked = gate.write(); - - // Drop the awaiting future mid-flight: the biased select! polls the - // put once — far enough to spawn the blocking task, which parks on - // the gate — then completes on the ready branch, dropping the put. - tokio::select! { - biased; - res = storage.put(&address, content) => { - panic!("put must be parked on the test gate, got {res:?}") - } - () = std::future::ready(()) => {} - } - - // The blocking op is still running: wait_idle must not complete. - let blocked = tokio::time::timeout(WAIT_IDLE_BLOCKED_PROBE, storage.wait_idle()).await; - assert!( - blocked.is_err(), - "wait_idle returned while the blocking op was parked" - ); - - // Release the gate: the detached closure commits and exits. - drop(parked); - tokio::time::timeout(WAIT_IDLE_COMPLETE_TIMEOUT, storage.wait_idle()) - .await - .expect("wait_idle after release"); - - // The dropped awaiter did not lose the write: it committed. - assert!(storage.exists(&address).expect("exists after release")); - - // The storage remains usable after wait_idle (tracker reopened). - let more = b"storage still usable after wait_idle"; - let more_addr = LmdbStorage::compute_address(more); - assert!(storage - .put(&more_addr, more) - .await - .expect("put after wait_idle")); - } - - /// A map resize takes the environment's *exclusive* lock, so it must wait - /// for in-flight raw reads (which hold the *shared* lock) to finish before - /// calling `env.resize()`. This proves `get_raw` holds that shared lock for - /// the whole duration of its blocking closure; `all_keys` uses the same - /// guard. - // Holding the gate's write guard across awaits is the point of the test: - // it parks the raw read's blocking closure while we probe try_resize. - #[allow(clippy::await_holding_lock)] - #[tokio::test] - async fn resize_waits_for_in_flight_raw_read() { - let (storage, _temp) = create_test_storage().await; - - let content = b"raw read holds the shared env lock"; - let address = LmdbStorage::compute_address(content); - storage.put(&address, content).await.expect("put"); - - // Park the raw read's blocking closure on the test gate. It acquires - // the shared env_lock first, then parks here still holding it. - let gate = storage.test_read_gate(); - let parked = gate.write(); - - // Drop the awaiting future mid-flight: the biased select! polls get_raw - // once — far enough to spawn the blocking task, which takes the shared - // lock and parks on the gate — then completes on the ready branch, - // dropping the awaiter. The detached closure keeps holding the lock. - tokio::select! { - biased; - res = storage.get_raw(&address) => { - panic!("get_raw must be parked on the test gate, got {res:?}") - } - () = std::future::ready(()) => {} - } - - // Wait until the detached read has actually taken the shared lock, - // signalled by the exclusive half no longer being immediately available. - tokio::time::timeout(RAW_READ_LOCK_TIMEOUT, async { - loop { - let free = storage.env_lock.try_write().is_some(); - if !free { - break; - } - tokio::time::sleep(RAW_READ_LOCK_POLL).await; - } - }) - .await - .expect("raw read did not take the shared env lock"); - - // A resize needs the exclusive lock, so it must block while the raw - // read holds the shared lock. - let resize = storage.try_resize(); - tokio::pin!(resize); - let blocked = tokio::time::timeout(RESIZE_BLOCKED_PROBE, &mut resize).await; - assert!( - blocked.is_err(), - "try_resize completed while a raw read held the shared env lock" - ); - - // Release the read: it drops the shared lock, letting the resize take - // the exclusive lock and finish. - drop(parked); - tokio::time::timeout(RESIZE_COMPLETE_TIMEOUT, &mut resize) - .await - .expect("try_resize did not complete after the raw read released") - .expect("try_resize"); - } - - /// The gate fires on `Full` and only on `Full`, so the verdict has to tell a - /// disk below its reserve from one above it, and has to notice when that - /// stops being true. - /// - /// The third call is the one that matters for recovery: the below-reserve - /// condition clears, and the verdict has to follow it rather than stay stuck - /// on its earlier answer. A node that remembered a refusal would stop - /// fetching for good. - /// - /// What this does not show: that a changed free-space reading is re-read from - /// the filesystem. The condition is cleared by dropping the reserve, which is - /// the same comparison approached from the other side. - #[tokio::test] - async fn capacity_verdict_follows_the_reserve_and_notices_recovery() { - let (writable, _temp) = create_test_storage().await; - assert_eq!(writable.capacity_verdict(), CapacityVerdict::Writable); - - let temp_dir = tempfile::TempDir::new().expect("create temp dir"); - let config = LmdbStorageConfig { - root_dir: temp_dir.path().to_path_buf(), - // Far above any real free space, the same way the e2e builds a - // write-blocked node. - disk_reserve: u64::MAX / 2, - ..LmdbStorageConfig::test_default() - }; - let mut full = LmdbStorage::new(config).await.expect("create storage"); - assert_eq!(full.capacity_verdict(), CapacityVerdict::Full); - - // Still short of space, so still `Full`. A `Full` result that populated - // the passing-result cache would answer `Writable` here. - assert_eq!(full.capacity_verdict(), CapacityVerdict::Full); - - // Space is no longer short. Nothing cached a refusal, so the very next - // read has to see it. - full.config.disk_reserve = 0; - assert_eq!( - full.capacity_verdict(), - CapacityVerdict::Writable, - "a refusal must not be negatively cached: the next read has to see the \ - below-reserve condition clear" - ); - } - - /// A space query that fails says nothing about available space, so it must - /// not read as a full disk. The gate stands a key down for five minutes on - /// `Full` alone, and a failed `statvfs` is not a condition worth standing - /// down for: it may be gone by the next cycle. - /// - /// The two `Ok` cases pin the boundary the reserve names: equal to the - /// reserve is writable, one byte under it is not. - /// - /// What this does not show: that the gate leaves `Unknown` alone. The gate - /// tests `== Full` on a separate line inside the verification cycle, which - /// needs a network to reach, so this covers the classification only. - #[test] - fn a_failed_space_query_reads_as_unknown_not_full() { - const RESERVE: u64 = 1024; - - assert_eq!( - verdict_from_available_space(Err(std::io::Error::other("space query failed")), RESERVE), - CapacityVerdict::Unknown, - "a failed query must not be reported as a full disk" - ); - - assert_eq!( - verdict_from_available_space(Ok(RESERVE - 1), RESERVE), - CapacityVerdict::Full - ); - assert_eq!( - verdict_from_available_space(Ok(RESERVE), RESERVE), - CapacityVerdict::Writable, - "at the reserve is not below it" - ); - } - - /// The verdict and the pre-check have to refuse on the same condition. - /// - /// They are two readings of one question — can this node write? — and two - /// callers depend on them separately: the verification cycle gates the - /// close-group probe on the verdict, while `execute_single_fetch` gates the - /// dial on the pre-check. A verdict stricter than the pre-check is the - /// harmful direction. A node the pre-check would let write stops - /// discovering holders for keys it could have stored, which is - /// under-replication rather than a saved probe, and nothing else in the - /// change would notice. - /// - /// This is a tripwire, deliberately built on the state where the two are - /// about to part company rather than on a bare full disk. ant-node - /// \#210 makes the pre-check a two-part predicate — below the reserve *and* - /// out of reusable pages inside the store — and a store that has deleted - /// more than one chunk's worth of pages fails only the first half, because - /// LMDB returns those pages to its own free list and never to the - /// filesystem. On a bare full disk the two predicates still agree, so a - /// test built on one would pass straight through the divergence. Whichever - /// of the two changes merges second has to carry the second half into the - /// verdict, and this is what makes that a red test rather than a textual - /// conflict resolved without it. - /// - /// What this does not show: which of `Full` and `Unknown` a refusal is. - /// `a_failed_space_query_reads_as_unknown_not_full` pins that, and only - /// `Full` reaches the gate. - #[tokio::test] - async fn capacity_verdict_refuses_exactly_when_check_capacity_does() { - let (mut storage, _temp) = create_test_storage().await; - assert_eq!(storage.capacity_verdict(), CapacityVerdict::Writable); - assert!( - storage.check_capacity().is_ok(), - "a writable verdict has to mean the pre-check admits the write" - ); - - // Two chunks written and deleted, so the store sits on more than one - // chunk of space LMDB can reuse and the filesystem will never take - // back. This is the pruned node the two predicates disagree about. - for fill in [1u8, 2u8] { - let content = vec![fill; MAX_CHUNK_SIZE]; - let address = LmdbStorage::compute_address(&content); - assert!(storage.put(&address, &content).await.expect("put")); - assert!(storage.delete(&address).await.expect("delete")); - } - - // Established without either function under test, so the scenario does - // not rest on the thing being measured. - storage.config.disk_reserve = u64::MAX / 2; - *storage.last_disk_ok.lock() = None; - let available = fs2::available_space(&storage.env_dir).expect("query available space"); - assert!( - available < storage.config.disk_reserve, - "the volume has to read as below the reserve, or neither predicate is \ - being asked the interesting question" - ); - // Reusable bytes read the way the two-part predicate reads them: the - // file's size less the pages still holding data. `Env::stat` rather - // than `non_free_pages_size`, which calls `String::from_utf8(key)` and - // unwraps, so it panics on our 32 random bytes of key. - let stat = storage.env.stat(); - let live_pages = (stat.branch_pages as u64) - .saturating_add(stat.leaf_pages as u64) - .saturating_add(stat.overflow_pages as u64); - let live_bytes = live_pages.saturating_mul(u64::from(stat.page_size)); - let file_bytes = storage.env.real_disk_size().expect("query store file size"); - let reusable = file_bytes.saturating_sub(live_bytes); - assert!( - reusable > MAX_CHUNK_SIZE as u64, - "the store has to sit on more than one chunk of reusable space, or the \ - two predicates are not yet being asked to differ \ - (file_bytes={file_bytes}, live_bytes={live_bytes})" - ); - - // Neither call caches a refusal, so the order of the two does not - // decide either answer. - let pre_check_refuses = storage.check_capacity().is_err(); - let verdict = storage.capacity_verdict(); - assert_eq!( - pre_check_refuses, - verdict != CapacityVerdict::Writable, - "the dial pre-check and the verification gate disagree about whether \ - this node can write: check_capacity refuses={pre_check_refuses}, \ - verdict={verdict:?}. A verdict of Full under a pre-check that admits \ - the write leaves a node that can store chunks refusing to look for \ - them" - ); - } - - // ── Capacity below the disk reserve (LMDB reuse) ──────────────────── - - /// Value size for the reuse tests. A whole number of chunks' worth, so the - /// space freed by a few deletes is unambiguously enough for one more. - const REUSE_VALUE_LEN: usize = 1024 * 1024; - - /// Distinct filler of `REUSE_VALUE_LEN` bytes. - fn reuse_filler(seed: u32) -> Vec { - let mut content = seed.to_le_bytes().to_vec(); - content.resize(REUSE_VALUE_LEN, 0u8); - content - } - - /// A config for `dir` whose reserve exceeds any real disk, so the store - /// always sees itself as below the reserve. - fn below_reserve_config(dir: &Path) -> LmdbStorageConfig { - LmdbStorageConfig { - root_dir: dir.to_path_buf(), - disk_reserve: u64::MAX, - ..LmdbStorageConfig::test_default() - } - } - - /// Write `count` chunks with an unconstrained reserve, returning their - /// addresses in insertion order. - async fn seed_chunks(dir: &Path, count: u32) -> Vec { - let config = LmdbStorageConfig { - root_dir: dir.to_path_buf(), - ..LmdbStorageConfig::test_default() - }; - let storage = LmdbStorage::new(config).await.expect("create storage"); - - let mut addresses = Vec::new(); - for seed in 0..count { - let content = reuse_filler(seed); - let address = LmdbStorage::compute_address(&content); - storage.put(&address, &content).await.expect("seed put"); - addresses.push(address); - } - - storage.wait_idle().await; - addresses - } - - fn file_len(storage: &LmdbStorage) -> u64 { - storage.env.real_disk_size().expect("real_disk_size") - } - - /// The regression this change is about: a node whose volume is below the - /// reserve must still write into pages an earlier delete freed. Before the - /// fix the pre-check refused on the disk half alone, so a node that had - /// pruned sat on reusable space it could not use. - #[tokio::test] - async fn below_reserve_put_reuses_freed_pages() { - let temp_dir = tempfile::TempDir::new().expect("create temp dir"); - let seeded = seed_chunks(temp_dir.path(), 12).await; - - let storage = LmdbStorage::new(below_reserve_config(temp_dir.path())) - .await - .expect("reopen storage"); - - let content = reuse_filler(u32::MAX); - let address = LmdbStorage::compute_address(&content); - - // Nothing freed yet, so this write would have to grow the file. Below - // the reserve that is exactly what must be refused. - assert!( - storage.put(&address, &content).await.is_err(), - "a write that must grow the file was allowed below the reserve" - ); - - // Free several chunks. Their pages go on LMDB's free list, not back to - // the filesystem, so `statvfs` still reports the volume as full. - for seeded_address in seeded.iter().take(6) { - assert!(storage.delete(seeded_address).await.expect("delete")); - } - - let before = file_len(&storage); - let stored = storage - .put(&address, &content) - .await - .expect("put into freed pages was refused below the reserve"); - assert!(stored); - assert_eq!( - storage.get(&address).await.expect("get"), - Some(content), - "chunk written into reused pages did not read back" - ); - - // The whole point: it was served from inside the existing file. - assert_eq!( - file_len(&storage), - before, - "reusing freed pages grew data.mdb, consuming the reserve" - ); - } - - /// A refused write must not have grown the file on its way to failing, - /// which is what protects the reserve while the map is pinned. - #[tokio::test] - async fn below_reserve_refused_put_does_not_grow_the_file() { - let temp_dir = tempfile::TempDir::new().expect("create temp dir"); - let _ = seed_chunks(temp_dir.path(), 6).await; - - let storage = LmdbStorage::new(below_reserve_config(temp_dir.path())) - .await - .expect("reopen storage"); - - let content = reuse_filler(u32::MAX); - let address = LmdbStorage::compute_address(&content); - - // Take the baseline after the first attempt, so it includes the pin. - let refusal = storage - .put(&address, &content) - .await - .expect_err("a write that must grow the file was allowed"); - assert!( - refusal.to_string().contains("Insufficient disk space"), - "refused for the wrong reason: {refusal}" - ); - let before = file_len(&storage); - let pinned_map = storage.env.info().map_size; - - for seed in 0..4u32 { - let content = reuse_filler(u32::MAX - 1 - seed); - let address = LmdbStorage::compute_address(&content); - let refusal = storage - .put(&address, &content) - .await - .expect_err("a write that must grow the file was allowed"); - assert!( - refusal.to_string().contains("Insufficient disk space"), - "refused for the wrong reason: {refusal}" - ); - } - - assert_eq!( - storage.env.info().map_size, - pinned_map, - "the map ceiling drifted while writes were being refused" - ); - - assert_eq!( - file_len(&storage), - before, - "refused writes still extended data.mdb into the reserve" - ); - } - - /// One refused maximum-sized value must not lock out smaller ones. LMDB's - /// `MapFull` is specific to the allocation it was asked for, so remembering - /// it store-wide would let a single large chunk deny every later write. - #[tokio::test] - async fn large_refusal_does_not_block_a_smaller_put() { - let temp_dir = tempfile::TempDir::new().expect("create temp dir"); - let seeded = seed_chunks(temp_dir.path(), 10).await; - - let storage = LmdbStorage::new(below_reserve_config(temp_dir.path())) - .await - .expect("reopen storage"); - - // Free room for a small value, but not for a large one. - let Some(first) = seeded.first() else { - panic!("seed_chunks returned no addresses"); - }; - assert!(storage.delete(first).await.expect("delete")); - - // A value far larger than what was freed cannot fit. - let oversized = vec![3u8; 8 * REUSE_VALUE_LEN]; - let oversized_address = LmdbStorage::compute_address(&oversized); - assert!(storage.put(&oversized_address, &oversized).await.is_err()); - - // A small value still must, using the pages the delete released. - let small = b"small record that fits in a freed page".to_vec(); - let small_address = LmdbStorage::compute_address(&small); - let stored = storage - .put(&small_address, &small) - .await - .expect("a large refusal blocked a small put that had room"); - assert!(stored); - } - - /// A store with no reusable page must still be able to delete, or it can - /// never prune its way back to health. - #[tokio::test] - async fn full_store_below_reserve_can_still_delete() { - let temp_dir = tempfile::TempDir::new().expect("create temp dir"); - let seeded = seed_chunks(temp_dir.path(), 8).await; - - let storage = LmdbStorage::new(below_reserve_config(temp_dir.path())) - .await - .expect("reopen storage"); - - // Pin the map by attempting a write that cannot fit. - let content = reuse_filler(u32::MAX); - let address = LmdbStorage::compute_address(&content); - assert!(storage.put(&address, &content).await.is_err()); - assert!(storage.no_growth.load(Ordering::Acquire)); - - let pinned_map = storage.env.info().map_size; - - for seeded_address in &seeded { - assert!( - storage.delete(seeded_address).await.expect("delete"), - "a pinned store could not prune" - ); - } - assert_eq!(storage.current_chunks().expect("current_chunks"), 0); - - // The allowance bounds permanent file growth, not the number of - // assisted deletes. Pruning a pinned store must stay possible however - // many deletes it takes, so whatever was charged has to be growth the - // file really took, and has to stay inside the budget. - let charged = storage.delete_growth_charged.load(Ordering::Acquire); - assert!( - charged < DELETE_COW_GROWTH_BUDGET, - "deletes exhausted the growth allowance ({charged} B) while pruning a pinned store" - ); - - // Whether or not any delete needed the maintenance allowance, none of - // it may be left in the ceiling afterwards: a raised ceiling is - // ordinary put capacity, so leaking it hands away the reserve. - let file_bytes = file_len(&storage); - assert!( - storage.env.info().map_size as u64 <= file_bytes.max(pinned_map as u64), - "delete left maintenance slack in the map ceiling" - ); - - // And the store must still refuse a write it cannot fit, i.e. the pin - // is still doing its job after the prune. - let oversized = vec![9u8; 64 * REUSE_VALUE_LEN]; - let oversized_address = LmdbStorage::compute_address(&oversized); - assert!( - storage.put(&oversized_address, &oversized).await.is_err(), - "pinning stopped working after a delete used the allowance" - ); - } - - /// The pre-check must admit while reuse is plausible and refuse once it is - /// not. Refusing on `statvfs` alone is what blinded a node to its own free - /// pages, so being below the reserve cannot by itself be an error. - #[tokio::test] - async fn check_capacity_tracks_reusable_space_not_just_disk() { - let temp_dir = tempfile::TempDir::new().expect("create temp dir"); - let seeded = seed_chunks(temp_dir.path(), 16).await; - - let storage = LmdbStorage::new(below_reserve_config(temp_dir.path())) - .await - .expect("reopen storage"); - - // A freshly written store has almost no free page, so below the reserve - // the pre-check refuses and the caller skips its expensive setup. - assert!( - storage.check_capacity().is_err(), - "pre-check stayed open on a store with no reusable space" - ); - - // Pruning puts pages back on the free list. Nothing is returned to the - // filesystem, so `statvfs` is unchanged and only the reusable half of - // the predicate can reopen the node. - for seeded_address in seeded.iter().take(10) { - assert!(storage.delete(seeded_address).await.expect("delete")); - } - - storage - .check_capacity() - .expect("pre-check stayed closed after pruning freed pages"); - } - - /// Freeing disk must lift the pin, or a node would stay clamped to its - /// high-water mark after an operator grew the partition. - #[tokio::test] - async fn leaving_no_growth_restores_head_room() { - let temp_dir = tempfile::TempDir::new().expect("create temp dir"); - let _ = seed_chunks(temp_dir.path(), 6).await; - - let mut storage = LmdbStorage::new(below_reserve_config(temp_dir.path())) - .await - .expect("reopen storage"); - - let content = reuse_filler(u32::MAX); - let address = LmdbStorage::compute_address(&content); - assert!(storage.put(&address, &content).await.is_err()); - assert!(storage.no_growth.load(Ordering::Acquire)); - let pinned_map = storage.env.info().map_size; - - // Simulate the operator freeing space: the reserve is now satisfiable. - storage.config.disk_reserve = 0; - *storage.last_disk_ok.lock() = None; - - let stored = storage - .put(&address, &content) - .await - .expect("store stayed pinned after disk was freed"); - assert!(stored); - assert!(!storage.no_growth.load(Ordering::Acquire)); - assert!( - storage.env.info().map_size > pinned_map, - "map was not re-grown after leaving no-growth mode" - ); - } - - /// Above the reserve nothing changes: no pinning, and writes grow the file - /// on demand exactly as before. - #[tokio::test] - async fn above_reserve_behaviour_is_unchanged() { - let (storage, _temp) = create_test_storage().await; - - storage - .check_capacity() - .expect("pre-check on a healthy node"); - - let content = reuse_filler(1); - let address = LmdbStorage::compute_address(&content); - assert!(storage.put(&address, &content).await.expect("put")); - assert!(!storage.no_growth.load(Ordering::Acquire)); - storage - .check_capacity() - .expect("pre-check after a healthy put"); - } -} diff --git a/src/storage/migration.rs b/src/storage/migration.rs deleted file mode 100644 index cd53a03f..00000000 --- a/src/storage/migration.rs +++ /dev/null @@ -1,3141 +0,0 @@ -//! Moving a node off LMDB and onto the file store without losing a chunk. -//! -//! LMDB never returns a deleted page to the filesystem. Disk comes back exactly once, -//! when `chunks.mdb` is removed whole, so a node cannot free space by deleting chunks -//! and cannot compact its way out either (compaction needs free space equal to the live -//! data, which is the same condition). That single fact shapes everything here. -//! -//! # The two ways this could lose data, and why neither can happen -//! -//! 1. **A node deletes its LMDB before the chunks are safely in files.** Retirement is -//! gated on the file store already holding every key the node still claims, and on -//! the existing retention contract: a key the node is still answerable for under a -//! gossiped commitment vetoes the delete. -//! 2. **Every node sheds the same chunk at once.** A node only sheds when it cannot fit -//! its own payload, and it sheds by close-group rank, furthest first. A chunk has -//! exactly one 7th-closest and one 6th-closest holder, so it is only ever a shed -//! candidate for two of its seven holders, and the staged rollout brings that to one. -//! -//! # Three releases -//! -//! Slashing is the *auditor's* decision, so a node cannot protect itself from being -//! penalised for a shed. Everyone else has to stop first, which is why this lands over -//! three releases rather than one: -//! -//! | Release | Penalise not holding a close-group chunk? | [`MigrationConfig::retire_legacy`] | -//! |---|---|---| -//! | First: stop that one penalty | no | `false` | -//! | Second: migrate | no | `true` | -//! | Third: restore it | yes | `true` | -//! -//! The penalty column is not a field here. It lives once, in -//! [`crate::replication::config::RELEASE_SUSPEND_CLOSE_GROUP_STORAGE_PENALTY`], and both -//! the auditors that apply it and the shedder that depends on it being withheld read that -//! same switch. Two copies would let a node shed while its peers still penalised. -//! -//! Audits keep running and keep recording throughout. What the first release withholds is narrow and -//! deliberate: only the penalty for *not holding a close-group chunk*. The -//! commitment-bound subtree audit still penalises in every release, because the whole -//! migration turns on a node's reduced commitment still being binding. The record those -//! audits keep is also how we will know when the third release is safe to ship. - -use crate::ant_protocol::XorName; -use crate::error::{Error, Result}; -use crate::logging::{debug, info, warn}; -use crate::replication::config::{storage_admission_width, ReplicationConfig}; -use crate::replication::pruning::{ - prove_peers_hold_records, prune_proofs_needed, target_peers_reported_present, -}; -use crate::storage::chunk_store::{ChunkStore, VerifyReport}; -use saorsa_core::identity::PeerId; -use saorsa_core::P2PNode; -use serde::{Deserialize, Serialize}; -use std::collections::{HashMap, HashSet}; -use std::path::{Path, PathBuf}; -use std::sync::Arc; -use std::time::{Duration, Instant}; -use std::time::{SystemTime, UNIX_EPOCH}; -use tokio_util::sync::CancellationToken; - -/// Filename of the persisted migration marker, under the node root. -pub const MIGRATION_STATE_FILE: &str = "migration-state.json"; - -/// Marker schema this build writes and understands. -const STATE_SCHEMA: u32 = 1; - -/// Floor on [`MigrationConfig::retire_delay_hours`]. -/// -/// `GOSSIP_ANSWERABILITY_TTL` is three hours at a one-hour rotation cadence, so a -/// commitment that named a shed key stops being answerable three hours after it was last -/// gossiped. Four hours clears that with an hour to spare. -pub const MIN_RETIRE_DELAY_HOURS: u64 = 4; - -/// How long between attempts to reopen an environment this node has lost its handle to. -/// -/// Each attempt scans every key in it, so retrying on every tick would spend a large store -/// entirely on failing to open it. -const HANDLE_RECOVERY_INTERVAL: Duration = Duration::from_mins(5); - -/// The longest one node may hold the volume migration lock before giving others a turn. -/// -/// Every branch that waits rather than works is meant to give the lock back on its own. -/// This is the backstop for the one that does not: without it, a node stuck on a condition -/// that never resolves stops every other node sharing the disk from ever starting, for the -/// whole release. Longer than a copy pass and a verification take, so it never interrupts -/// a node that is genuinely working. -const MAX_VOLUME_LOCK_HOLD: Duration = Duration::from_hours(6); - -/// How long a node stands back after the cap takes the volume lock off it. -/// -/// Long enough that another node waiting on the lock actually gets it, rather than losing -/// the race to the node that has just been holding it for six hours. -const VOLUME_LOCK_COOLDOWN: Duration = Duration::from_mins(2); - -/// How many commitment rebuilds must be observed after the node commits to its -/// file-backed set before the legacy environment may be retired. -/// -/// One proves the builder read the new set. Two proves it published and survived a -/// rotation, which is what makes the retention window meaningful. -pub const REQUIRED_REBUILDS_BEFORE_RETIRE: u32 = 2; - -/// Operator-facing controls for the migration. -// Four independent switches, three of which are operator controls and one of which is a -// release constant. Collapsing them into an enum would tie choices together that are -// deliberately separate. -#[allow(clippy::struct_excessive_bools)] -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct MigrationConfig { - /// Run the background copier at all. - /// - /// Turning this off leaves a node reading the union of both stores forever. It never - /// frees the LMDB's disk, so it is an escape hatch rather than a supported mode. - #[serde(default = "default_true")] - pub enabled: bool, - - /// Write every new chunk to the legacy environment as well as the file store. - /// - /// Costs roughly a gigabyte per node per month at the observed fill rate, and buys - /// the ability to roll the fleet back: a chunk uploaded during the bridge to holders - /// that all revert to a pre-migration build would otherwise be gone from every one - /// of them. Automatically irrelevant once the legacy environment is retired. - #[serde(default = "default_true")] - pub dual_write_legacy: bool, - - /// Allow a node that cannot fit its payload to drop its furthest keys. - /// - /// An operator who would rather add disk than shed can set this to `false`. The node - /// then keeps both stores and never frees the LMDB's space. - #[serde(default = "default_true")] - pub allow_shed: bool, - - /// Delete `chunks.mdb` once the retirement gate is satisfied. - /// - /// **On in this release**, because it is the only step that returns disk. It is also - /// the only destructive step in the whole migration and the only one that cannot be - /// undone, which is why everything in front of it is a gate: the wave the node is - /// assigned to, the shed hold, the reduced commitment reaching the close group, - /// possession of every chunk being given up proven elsewhere, a re-read of every - /// remaining chunk, and a retention delay on top. - /// - /// Deliberately never serialised. A node writes its effective configuration back to - /// disk, so shipping this as an ordinary field would bake R1's `false` into every - /// operator's config file and R2 would then never retire anything. The release phase - /// belongs to the build, not to the operator's file. `ANT_MIGRATION_RETIRE_LEGACY` - /// overrides it for a canary. - #[serde(skip, default = "release_retire_legacy")] - pub retire_legacy: bool, - - /// Hours after this build first starts before a node may shed anything. - /// - /// Long enough for peers still on a pre-R1 build to upgrade, because one of those - /// still penalises a shedder at the full audit weight. - #[serde(default = "default_shed_hold_hours")] - pub shed_hold_hours: u64, - - /// Hours between committing to the file-backed key set and deleting `chunks.mdb`. - /// - /// Clamped up to [`MIN_RETIRE_DELAY_HOURS`]. Longer buys a rollback window on nodes - /// that can afford to hold both copies. - #[serde(default = "default_retire_delay_hours")] - pub retire_delay_hours: u64, - - /// Free megabytes the copier leaves untouched, on top of the disk reserve. - /// - /// The copier stops here rather than filling to the brink, so a node that is - /// mid-migration still has room to accept a chunk it is paid for. - #[serde(default = "default_copier_slack_mb")] - pub copier_slack_mb: u64, - - /// Copy rate ceiling, in mebibytes per second. - /// - /// The quiet responsible audit lane is where audit timeouts actually cost trust, and - /// an unthrottled copier competing with it for I/O is the fastest way to turn a - /// storage migration into an audit incident. - #[serde(default = "default_copier_throttle_mib_per_sec")] - pub copier_throttle_mib_per_sec: u64, - - /// Hours between one migration wave opening and the next. - /// - /// A close group is split into waves so that only - /// [`CONCURRENT_MIGRATIONS_PER_GROUP`] of it give chunks up at a time. This is how - /// long a wave gets to finish copying, retiring and refetching before the next one may - /// start. Only nodes that have to give something up wait for their wave; a node with - /// room migrates immediately. - #[serde(default = "default_wave_hours")] - pub wave_hours: u64, - - /// Seconds between copier ticks. - #[serde(default = "default_tick_secs")] - pub tick_secs: u64, - - /// Chunks copied per tick before yielding. - #[serde(default = "default_batch_chunks")] - pub batch_chunks: usize, - - /// Where the volume lock lives, overriding the filesystem this node's root sits on. - /// - /// `None` in production, which keys the lock by device id so every node on one disk - /// serialises against the others. Tests set it so a test's migration contends only - /// with its own, rather than with every other test sharing the machine's filesystem. - /// - /// Never serialised: it exists to scope a test, not to configure a node. - #[serde(skip)] - pub lock_dir: Option, -} - -const fn default_true() -> bool { - true -} - -/// Whether this build deletes the legacy environment once the gate is satisfied. -/// -/// **`true` in this release.** Deleting `chunks.mdb` is the only step that returns disk, -/// and a build that ships with it off is a migration that never finishes: the fleet -/// already deleted 2.29M chunks out of LMDB and got back nothing, because LMDB does not -/// return freed pages to the filesystem. Every gate in front of this is still enforced, -/// and `ANT_MIGRATION_RETIRE_LEGACY=0` turns it off on a single node if one is ever -/// needed to hold both stores. -pub const RELEASE_RETIRE_LEGACY: bool = true; - -/// Environment override for the directory the per-volume migration lock lives in. -/// -/// Set this where the default cannot work, and the default cannot work wherever the nodes -/// sharing a disk do not share a `/tmp`. Our own multi-node hosts are exactly that case: -/// the systemd unit sets `PrivateTmp=true`, which gives every unit a tmpfs of its own, so -/// each node creates the same lock filename in a different filesystem, every one of them -/// takes it, and the lock serialises nothing. Point every node on a host at one directory -/// they can all write and the lock does what it is for. -/// -/// The directory must be writable by the node. A path that cannot be used is reported and -/// the node migrates unserialised, which is the same answer as having no lock, so a typo -/// here is loud rather than silent. -pub const LOCK_DIR_ENV: &str = "ANT_MIGRATION_LOCK_DIR"; - -/// Environment override for [`RELEASE_RETIRE_LEGACY`], for a canary node. -pub const RETIRE_LEGACY_ENV: &str = "ANT_MIGRATION_RETIRE_LEGACY"; - -/// Read a boolean override from the environment, falling back to the build constant. -fn env_override(name: &str, build_default: bool) -> bool { - let Ok(raw) = std::env::var(name) else { - return build_default; - }; - match raw.trim().to_ascii_lowercase().as_str() { - "1" | "true" | "yes" | "on" => true, - "0" | "false" | "no" | "off" => false, - other => { - warn!("{name}={other} is not a boolean; using the build default {build_default}"); - build_default - } - } -} - -/// The retirement switch for this build, after any environment override. -fn release_retire_legacy() -> bool { - env_override(RETIRE_LEGACY_ENV, RELEASE_RETIRE_LEGACY) -} - -const fn default_shed_hold_hours() -> u64 { - 72 -} - -const fn default_retire_delay_hours() -> u64 { - MIN_RETIRE_DELAY_HOURS -} - -const fn default_wave_hours() -> u64 { - 24 -} - -const fn default_copier_slack_mb() -> u64 { - 2048 -} - -const fn default_copier_throttle_mib_per_sec() -> u64 { - 32 -} - -const fn default_tick_secs() -> u64 { - 30 -} - -const fn default_batch_chunks() -> usize { - 64 -} - -impl Default for MigrationConfig { - fn default() -> Self { - Self { - enabled: true, - dual_write_legacy: true, - allow_shed: true, - retire_legacy: release_retire_legacy(), - shed_hold_hours: default_shed_hold_hours(), - retire_delay_hours: default_retire_delay_hours(), - wave_hours: default_wave_hours(), - copier_slack_mb: default_copier_slack_mb(), - copier_throttle_mib_per_sec: default_copier_throttle_mib_per_sec(), - tick_secs: default_tick_secs(), - batch_chunks: default_batch_chunks(), - lock_dir: None, - } - } -} - -impl MigrationConfig { - /// The retire delay, never shorter than the retention contract allows. - #[must_use] - pub fn effective_retire_delay_hours(&self) -> u64 { - self.retire_delay_hours.max(MIN_RETIRE_DELAY_HOURS) - } - - /// Copier slack in bytes. - #[must_use] - pub fn copier_slack_bytes(&self) -> u64 { - self.copier_slack_mb.saturating_mul(1024 * 1024) - } -} - -/// Where a node is in the migration. -/// -/// The phase is persisted, but only as a *decision* record. Everything derivable from -/// the filesystem is re-derived at every start: which keys are still legacy-only is -/// simply "in the LMDB and not in the file store", so an interrupted copy resumes for -/// free with no progress bookkeeping to corrupt. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum MigrationPhase { - /// Copying the legacy environment into files. Reads are the union of both stores, - /// writes go to files, and the commitment still covers everything. - Bridging, - /// The node has settled on what it will keep and commits only to its file-backed - /// keys. It keeps serving the rest from LMDB until they stop being answerable. - Committed, - /// No legacy environment. Steady state, and where every fresh node starts. - FilesOnly, -} - -/// The persisted migration marker. -/// -/// Two facts genuinely need to survive a restart: when this build first ran (so the shed -/// hold is not restarted by a reboot loop) and when the node committed to its file-backed -/// set (so the retirement clock is not either). Everything else is re-derived. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct MigrationState { - /// Marker schema version. - pub schema: u32, - /// Where the node is. - pub phase: MigrationPhase, - /// Unix seconds when this build first started on this node. - pub first_start_unix: u64, - /// Unix seconds when the node committed to its file-backed key set. - pub committed_at_unix: Option, - /// Commitment rebuilds observed since committing. - pub rebuilds_since_commit: u32, - /// How many keys the node decided not to keep, for the operator's benefit. - pub shed_key_count: u64, - /// How many chunks the file store held when the node committed. - /// - /// Cross-checked at open. A marker claiming the node is past the copying stage while - /// the file store is far emptier than it said means the two disagree about reality, - /// and the filesystem wins. - #[serde(default)] - pub kept_key_count: u64, -} - -impl MigrationState { - /// A fresh marker for a node that has just started. - #[must_use] - pub fn new(phase: MigrationPhase) -> Self { - Self { - schema: STATE_SCHEMA, - phase, - first_start_unix: now_unix(), - committed_at_unix: None, - rebuilds_since_commit: 0, - shed_key_count: 0, - kept_key_count: 0, - } - } - - /// Load the marker, writing a fresh one if there is none. - /// - /// Persisting immediately matters: `first_start_unix` is what the shed hold counts - /// from, and a marker that is only written at the first phase change would reset that - /// clock on every restart before then, so a node that restarts more often than the - /// hold would never become eligible to shed and never finish migrating. - pub fn load_or_create(root_dir: &Path, phase: MigrationPhase) -> Self { - let state = Self::load_or_new(root_dir, phase); - // Written whenever the disk does not already hold what this process is going to - // use, rather than only when there is no file at all. A marker that is present but - // could not be used is replaced in memory and was previously left on disk, so the - // next start read the same bad file and stamped `first_start_unix` afresh. That is - // precisely the reset this function exists to prevent, and it is worse than the - // one it does prevent: it repeats. Three ways in, all of them leaving a file that - // exists: a truncated or otherwise unparseable marker, one from a newer schema, and - // one whose clock is impossible and is corrected by `with_sane_clocks`. - // - // A node restarting more often than the shed hold then never becomes eligible to - // shed and never finishes migrating, and a node short of disk is exactly the node - // that restarts. - let raw = std::fs::read(state_path(root_dir)).ok(); - let on_disk = raw - .as_deref() - .and_then(|bytes| serde_json::from_slice::(bytes).ok()); - if on_disk.as_ref() == Some(&state) { - return state; - } - // The schema is read on its own, from the raw JSON, rather than taken from a - // successful parse into today's struct. A marker from a genuinely newer build is - // exactly the one least likely to parse into it: a phase this build has no name - // for, a field that changed type, a field that went away. Reading the schema only - // when the whole thing parses means the markers most worth keeping are the ones - // that would be written over. - let newer_schema = raw - .as_deref() - .and_then(|bytes| serde_json::from_slice::(bytes).ok()) - .and_then(|value| value.get("schema").and_then(serde_json::Value::as_u64)) - .filter(|schema| *schema > u64::from(STATE_SCHEMA)); - if let Some(schema) = newer_schema { - let Some(kept) = free_kept_marker_path(root_dir, schema) else { - warn!( - "A newer migration marker (schema {schema}) is here and every name to \ - keep it under is taken. Leaving it, which means the shed hold restarts \ - on every boot until it is dealt with" - ); - return state; - }; - if let Err(e) = std::fs::rename(state_path(root_dir), &kept) { - warn!( - "Could not move the newer migration marker aside ({e}); leaving it, \ - which means the shed hold restarts on every boot until it is dealt with" - ); - return state; - } - info!( - "Kept the newer migration marker as {} and started one this build can use", - kept.display() - ); - } - if let Err(e) = state.save(root_dir) { - warn!("Could not write the migration marker: {e}"); - } - state - } - - /// Load the marker, or start a fresh one. - /// - /// An unreadable marker is replaced rather than fatal: it is a hint, and every fact - /// it holds is either recoverable or conservative to reset. Losing it restarts the - /// shed hold and the retirement clock, which delays a migration and never rushes one. - pub fn load_or_new(root_dir: &Path, phase: MigrationPhase) -> Self { - let path = state_path(root_dir); - let Ok(bytes) = std::fs::read(&path) else { - return Self::new(phase); - }; - match serde_json::from_slice::(&bytes) { - Ok(state) if state.schema <= STATE_SCHEMA => state.with_sane_clocks(), - Ok(state) => { - warn!( - "Migration marker {} was written by a newer build (schema {}); \ - starting a fresh one", - path.display(), - state.schema - ); - Self::new(phase) - } - Err(e) => { - warn!( - "Migration marker {} is unreadable ({e}); starting a fresh one. \ - The shed hold and retirement clock restart from now.", - path.display() - ); - Self::new(phase) - } - } - } - - /// Persist the marker so a reader sees either the old content or the new. - /// - /// # Errors - /// - /// Returns [`Error::Storage`] if the marker cannot be written. - pub fn save(&self, root_dir: &Path) -> Result<()> { - let path = state_path(root_dir); - let bytes = serde_json::to_vec_pretty(self) - .map_err(|e| Error::Storage(format!("Failed to encode migration marker: {e}")))?; - crate::storage::file_store::write_file_durably(&path, &bytes)?; - debug!("Migration marker updated: phase {:?}", self.phase); - Ok(()) - } - - /// Replace timestamps that cannot be true with "now". - /// - /// Zero and future values both make a hold vacuous, and a node whose clock was not - /// yet synchronised at first boot writes zero without anyone tampering. Resetting to - /// now delays a migration, which is the safe direction. - #[must_use] - fn with_sane_clocks(mut self) -> Self { - let now = now_unix(); - if self.first_start_unix == 0 || self.first_start_unix > now { - warn!("Migration marker has an implausible first-start time; restarting the hold"); - self.first_start_unix = now; - } - self.committed_at_unix = self.committed_at_unix.map(|at| { - if at == 0 || at > now { - warn!("Migration marker has an implausible commit time; restarting the clock"); - now - } else { - at - } - }); - self - } - - /// Whether the shed hold has elapsed. - #[must_use] - pub fn shed_hold_elapsed(&self, config: &MigrationConfig) -> bool { - let hold = config.shed_hold_hours.saturating_mul(3600); - now_unix().saturating_sub(self.first_start_unix) >= hold - } - - /// Whether the retirement delay has elapsed since committing. - #[must_use] - pub fn retire_delay_elapsed(&self, config: &MigrationConfig) -> bool { - let Some(at) = self.committed_at_unix else { - return false; - }; - let delay = config.effective_retire_delay_hours().saturating_mul(3600); - now_unix().saturating_sub(at) >= delay - } -} - -/// A name to keep a newer marker under that nothing is using yet. -/// -/// The plain `.schema-N` name is deterministic, so a node downgraded twice would otherwise -/// write over the marker it kept the first time, or fail the rename and restart the hold on -/// every boot. Returns `None` if every name is taken, which the caller reports rather than -/// destroying anything. -fn free_kept_marker_path(root_dir: &Path, schema: u64) -> Option { - for attempt in 0..16u32 { - // Appended, not `with_extension`, which would replace the `.json` and leave a name - // that no longer says what the file is. - let mut candidate = state_path(root_dir).into_os_string(); - candidate.push(format!(".schema-{schema}")); - if attempt > 0 { - candidate.push(format!(".{attempt}")); - } - let candidate = PathBuf::from(candidate); - // `symlink_metadata`, not `try_exists`. The latter follows links, so a dangling - // symbolic link at this name reads as nothing being there while `rename` would - // happily replace it. Anything at all, of any kind, means pick another name. - if std::fs::symlink_metadata(&candidate).is_err() { - return Some(candidate); - } - } - None -} - -/// Path of the persisted marker. -#[must_use] -pub fn state_path(root_dir: &Path) -> PathBuf { - root_dir.join(MIGRATION_STATE_FILE) -} - -/// Seconds since the Unix epoch, saturating at zero if the clock is before it. -#[must_use] -pub fn now_unix() -> u64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .map_or(0, |d| d.as_secs()) -} - -/// A host-wide advisory lock that serialises migrations sharing one volume. -/// -/// Twelve nodes on one 492 GiB volume each need roughly their live payload free to copy -/// and each return rather more when they retire, so one at a time the host gains space -/// and the queue accelerates. All twelve at once need twelve times the space and all -/// twelve stall. The lock is taken non-blocking: a node that cannot get it simply waits -/// for the next tick. -/// -/// Where the lock file sits is `lock_path_for`'s decision (private, so this is not a -/// link), and this doc used to describe -/// a branch of it that a running node almost never reaches: the parent of the node root is -/// the last resort, taken only when the root's own metadata cannot be read. What a node -/// normally uses is the host's temporary directory keyed by the volume's device id, or the -/// directory named by [`LOCK_DIR_ENV`] when one is set. -/// -/// Which of those is right is a fact about the deployment that no node can check for -/// itself, and getting it wrong is silent: every node takes a lock of its own and reports -/// success. That is why the path is logged when the lock is taken, and why a host whose -/// nodes do not share a `/tmp` has to be told where the lock lives. -/// -/// The directory has to be one only the node's own user can write. A predictable path in a -/// world-writable `/tmp` can be created and held by any local user, who could then keep -/// every node on the host from ever migrating. -#[derive(Debug)] -pub struct VolumeLock { - /// The held file. Dropping it releases the lock. - file: std::fs::File, - /// Where it lives, for logging. - #[cfg_attr(not(feature = "logging"), allow(dead_code))] - path: PathBuf, -} - -/// The result of asking for the volume lock. -pub enum LockAttempt { - /// This node has it. - Acquired(VolumeLock), - /// Another node on the volume is migrating. Wait. - Busy, - /// No lock is possible here at all, so proceed unserialised. - /// - /// Kept distinct from `Busy` because conflating the two silently strands any node - /// whose parent directory is not writable: it would wait forever for a lock nobody - /// holds. - Unavailable, -} - -impl VolumeLock { - /// Try to take the lock for the volume hosting `root_dir`. - #[must_use] - pub fn try_acquire(root_dir: &Path, scope: Option<&Path>) -> LockAttempt { - use fs2::FileExt; - let path = scope.map_or_else( - || lock_path_for(root_dir), - |dir| dir.join("ant-migration.lock"), - ); - let file = match std::fs::OpenOptions::new() - .write(true) - .create(true) - .truncate(false) - .open(&path) - { - Ok(f) => f, - Err(e) => { - warn!( - "Could not create the migration lock at {}: {e}. This node will \ - migrate without serialising against others on the same volume, so \ - watch its free space.", - path.display() - ); - return LockAttempt::Unavailable; - } - }; - match file.try_lock_exclusive() { - Ok(()) => { - // Info, not debug. Whether the lock is doing anything depends on whether - // the neighbours on this disk can see the same path, which is a deployment - // fact no node can check. Printing the path is what lets somebody answer it - // from a log rather than by reading a unit file. - info!("Took the volume migration lock at {}", path.display()); - LockAttempt::Acquired(Self { file, path }) - } - // Only contention means another node is migrating. Everything else, a - // filesystem that does not implement locking at all being the one that - // matters, is a lock this node will never get, and reporting it as contention - // would leave it waiting forever for a holder that does not exist. - Err(e) if is_lock_contention(&e) => LockAttempt::Busy, - Err(e) => { - warn!( - "Could not lock {}: {e}. This node will migrate without serialising \ - against others on the same volume, so watch its free space.", - path.display() - ); - LockAttempt::Unavailable - } - } - } -} - -/// Is this the error a lock held by someone else produces? -fn is_lock_contention(e: &std::io::Error) -> bool { - e.kind() == std::io::ErrorKind::WouldBlock - || (e.raw_os_error().is_some() - && e.raw_os_error() == fs2::lock_contended_error().raw_os_error()) -} - -/// Where the lock for the volume hosting `root_dir` lives. -/// -/// Keyed by the filesystem, not by the path. Two nodes on one host are configured with -/// different roots by definition, so a lock beside the root serialises a node against -/// nobody: `/srv/node-a/data` and `/srv/node-b/data` would take two different locks on one -/// disk and copy at the same time, which is the case the lock exists to prevent. -/// -/// The device id names the filesystem, and the host's temporary directory is somewhere -/// every node on that host can reach. If the device cannot be read, this falls back to a -/// lock beside the root: weaker, but never worse than having none. -/// -/// **That last sentence is only true where the nodes share a `/tmp`.** Where they do not, -/// each computes the same name in a filesystem of its own, every one of them takes it, and -/// the lock serialises nothing while logging that it worked. `PrivateTmp=true` in a systemd -/// unit does exactly that, and our own worker unit sets it. There is no way to tell from -/// inside one process whether the `/tmp` it can see is the one its neighbours see, so this -/// cannot be detected here and has to be configured: [`LOCK_DIR_ENV`] names a directory -/// every node on the host can reach, and is consulted first. -fn lock_path_for(root_dir: &Path) -> PathBuf { - lock_path_with(root_dir, std::env::var(LOCK_DIR_ENV).ok().as_deref()) -} - -/// The same decision, with the configured directory passed in rather than read. -/// -/// Split so it can be tested without touching process-wide environment. A test that set -/// `TMPDIR` to stage the private-`/tmp` case would change where every other test's -/// `TempDir` lands, and then delete it underneath them: run in parallel that takes out -/// dozens of unrelated tests with LMDB failures that look like anything but their cause. -fn lock_path_with(root_dir: &Path, configured: Option<&str>) -> PathBuf { - // Before anything derived, because an operator who has set this knows something about - // the host that this function cannot find out. - if let Some(dir) = configured { - let dir = dir.trim(); - if !dir.is_empty() { - return Path::new(dir).join("ant-migration.lock"); - } - } - #[cfg(unix)] - { - use std::os::unix::fs::MetadataExt; - if let Ok(meta) = std::fs::metadata(root_dir) { - return std::env::temp_dir().join(format!("ant-migration-{}.lock", meta.dev())); - } - } - // Resolved first, because a relative root has no volume in it to read. Two nodes - // started from different working directories on one drive would otherwise each fall - // through to a lock beside their own root, which serialises neither against the other. - #[cfg(not(unix))] - let resolved = std::fs::canonicalize(root_dir).unwrap_or_else(|_| root_dir.to_path_buf()); - #[cfg(not(unix))] - let root_dir = resolved.as_path(); - // Off Unix, the volume root: the drive or share the path starts from. Not as precise - // as a device id, since a mount point below it belongs to another volume, but it - // groups the ordinary case of several nodes under one drive letter, which is what a - // lock beside each node's own root does not. - #[cfg(not(unix))] - { - use std::path::Component; - if let Some(Component::Prefix(prefix)) = root_dir.components().next() { - let key: String = prefix - .as_os_str() - .to_string_lossy() - .chars() - .filter(char::is_ascii_alphanumeric) - .collect(); - if !key.is_empty() { - return std::env::temp_dir().join(format!("ant-migration-{key}.lock")); - } - } - } - root_dir - .parent() - .unwrap_or(root_dir) - .join("ant-migration.lock") -} - -impl Drop for VolumeLock { - fn drop(&mut self) { - use fs2::FileExt; - if let Err(e) = FileExt::unlock(&self.file) { - debug!( - "Releasing the migration lock {} failed: {e}", - self.path.display() - ); - } - } -} - -/// Summary of what the copier moved during one pass. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -pub struct CopyReport { - /// Chunks copied into the file store. - pub copied: u64, - /// Bytes copied. - pub bytes: u64, - /// Keys skipped because the legacy bytes did not hash to their address, or were - /// larger than a chunk may be. - pub unusable: u64, - /// Keys that could not be copied for a reason that may clear on a later pass. - pub failed: u64, - /// Keys that had vanished from the legacy store between the scan and the copy. - pub vanished: u64, - /// Whether the pass stopped because free space reached the slack floor. - pub stopped_for_space: bool, -} - -impl CopyReport { - /// Fold another pass into this one. - pub fn merge(&mut self, other: Self) { - self.copied += other.copied; - self.bytes += other.bytes; - self.unusable += other.unusable; - self.failed += other.failed; - self.vanished += other.vanished; - self.stopped_for_space |= other.stopped_for_space; - } -} - -/// How many waves a close group is divided into, so that at most -/// [`CONCURRENT_MIGRATIONS_PER_GROUP`] of it are giving chunks up at once. -#[must_use] -pub fn migration_wave_count(close_group_size: usize) -> u64 { - let group = close_group_size.max(1) as u64; - let per_wave = CONCURRENT_MIGRATIONS_PER_GROUP.max(1) as u64; - group.div_ceil(per_wave).max(1) -} - -/// Which wave this node belongs to, derived from its own ID. -/// -/// Deterministic and needs no coordination, which is the point: a node cannot ask its -/// close group "are you migrating?" without a protocol change, and the answer would be -/// stale by the time it arrived. Hashing the peer ID spreads the members of any group -/// across the waves without anybody agreeing on anything. -/// -/// It is a stagger, not a guarantee. Seven IDs hashed into four waves will not always land -/// two, two, two, one. What makes it safe rather than merely tidy is that it composes with -/// the possession gate: a node whose turn has come still cannot give a chunk up until its -/// neighbours have proven they hold it, so an unlucky wave waits instead of over-shedding. -#[must_use] -pub fn migration_wave_for(self_id: Option<&PeerId>, close_group_size: usize) -> u64 { - let waves = migration_wave_count(close_group_size); - let Some(peer) = self_id else { - return 0; - }; - let digest = blake3::hash(&[MIGRATION_WAVE_DOMAIN, peer.as_bytes().as_slice()].concat()); - let mut head = [0u8; 8]; - head.copy_from_slice(digest.as_bytes().get(..8).unwrap_or(&[0u8; 8])); - u64::from_le_bytes(head) % waves -} - -/// Domain separator so the wave assignment cannot be confused with any other use of a -/// hashed peer ID. -const MIGRATION_WAVE_DOMAIN: &[u8] = b"ant-node/storage-migration-wave/v1"; - -/// Whether this node's wave has opened yet. -/// -/// Wave `w` opens `w * wave_hours` after this build first started on this node. A node -/// that has room for everything never consults this: it copies and retires without ever -/// being unable to serve, so it is not part of the problem the waves exist to solve. -#[must_use] -pub fn wave_has_opened(state: &MigrationState, config: &MigrationConfig, wave: u64) -> bool { - now_unix() >= wave_opens_at(state, config, wave) -} - -/// When a given wave opens, in Unix seconds. -/// -/// Measured from the END of the shed hold, not from first start. Measured from the start -/// the two settings cancel each other out: with a 72 hour hold and 24 hour waves, waves -/// would open at 0, 24, 48 and 72 hours while nothing at all may shed until hour 72, so -/// every wave would be open the moment the first one could act and the whole close group -/// would migrate together. That is the pile-up the waves exist to prevent. -#[must_use] -pub fn wave_opens_at(state: &MigrationState, config: &MigrationConfig, wave: u64) -> u64 { - state - .first_start_unix - .saturating_add(config.shed_hold_hours.saturating_mul(3600)) - .saturating_add(wave.saturating_mul(config.wave_hours.saturating_mul(3600))) -} - -/// Order keys closest-first by XOR distance from this node. -/// -/// Shedding walks this list from the far end, so the keys a node gives up are the ones it -/// is furthest from, and therefore the ones its close group covers best. -#[must_use] -pub fn rank_closest_first(mut keys: Vec, self_xor: Option) -> Vec { - let Some(me) = self_xor else { - // No identity available (devnet, unit tests). Ascending key order is stable and - // deterministic, which is all the copier needs. - keys.sort_unstable(); - return keys; - }; - keys.sort_unstable_by_key(|k| crate::client::xor_distance(k, &me)); - keys -} - -/// Structured field marking every line the fleet gate for R3 is read from. -/// -/// R3 ships when the fleet shows migrations have finished, so these lines have to be -/// queryable rather than merely readable. One field name, three values. -pub const MIGRATION_EVENT: &str = "migration_event"; - -/// Log the operator-facing summary of a completed migration. -/// -/// `freed_bytes` is what the retired environment held, which is what the deletion running -/// in the background will return. The line that says the space is actually back is -/// `migration_event = "space_returned"`, emitted by that deletion when it finishes. Two -/// lines rather than one because the deletion of a large environment takes minutes, and a -/// node that reports the disk back before it is back is a node whose operator cannot tell -/// a slow deletion from a failed one. -pub fn log_migration_complete(kept: u64, shed: u64, freed_bytes: u64) { - #[allow(clippy::cast_precision_loss)] // display only - let freed_gib = freed_bytes as f64 / (1024.0 * 1024.0 * 1024.0); - if shed == 0 { - info!( - migration_event = "complete", - kept, - shed, - freed_bytes, - "Storage migration complete: {kept} chunks now in the file store, nothing shed, \ - {freed_gib:.2} GiB being returned to the filesystem" - ); - } else { - info!( - migration_event = "complete", - kept, - shed, - freed_bytes, - "Storage migration complete: kept {kept} chunks, shed {shed} that would not fit, \ - {freed_gib:.2} GiB being returned to the filesystem. The shed keys are the ones this \ - node was furthest from; replication will refetch what still belongs here now \ - that there is room." - ); - } -} - -// ──────────────────────────────────────────────────────────────────────────── -// The driver -// ──────────────────────────────────────────────────────────────────────────── - -/// How many positions from the end of the admission group a node may give up. -/// -/// A chunk has exactly one holder at each rank, so restricting shedding to the last two -/// positions means only two of its holders ever consider dropping it, and the staged -/// rollout brings that to one. Without this rule the property is only statistical: -/// every holder could be short of space at once, each shed the same chunk, and the -/// per-volume lock would not know, because it serialises one volume and this is a -/// network-wide question. -/// -/// Measured against [`storage_admission_width`], not the close group, so the migration is -/// never more willing to drop a chunk than the pruner is. The pruner treats the wider -/// group as strictly in-range and refuses to delete inside it; shedding ranks that the -/// pruner protects would make a one-off migration weaker than the thing that runs every -/// day. -pub const SHEDDABLE_TAIL_RANKS: usize = 2; - -/// How many nodes of one close group may be giving chunks up at the same time. -/// -/// The close group is the unit that matters, not the volume and not the fleet. If every -/// holder of a chunk migrates at once, none of them can prove to the others that a copy -/// survives, and the whole group deadlocks waiting on each other. Holding it to two means -/// the other five are steady, can answer possession challenges, and are still serving the -/// chunk while the two rebuild. -pub const CONCURRENT_MIGRATIONS_PER_GROUP: usize = 2; - -/// How many keys a refusal names, so the log stays readable. -const REFUSAL_SAMPLE: usize = 4; - -/// How recently a peer must have published a commitment to be trusted as a holder. -/// -/// Commitments rotate hourly and are gossiped on the neighbour-sync cadence, so a peer -/// that has not published one for this long is not simply quiet: it has either stopped -/// speaking the protocol or retired its commitment and not yet rotated a new one. The -/// second is exactly what a node in the middle of its own migration looks like, and -/// counting it as a holder is how two migrating nodes could each conclude the other was -/// covering the chunk. -const COMMITMENT_FRESHNESS: Duration = Duration::from_hours(2); - -/// How many keys one possession round asks about. -/// -/// The round batches by peer, so this bounds the size of a single request rather than the -/// number of requests. -const POSSESSION_BATCH_KEYS: usize = 256; - -/// How long to wait before re-evaluating a shed decision that was refused. -const SHED_REEVALUATION_INTERVAL: Duration = Duration::from_mins(10); - -/// How many copied chunks between operator-facing progress lines. -const PROGRESS_LOG_EVERY: usize = 500; - -/// How long a clean pre-retirement verification stays usable. -/// -/// Chunks written since the pass were content-checked on the way in and flushed, so the -/// only thing the window exposes is bit rot in the last half hour, which is the ordinary -/// risk of any file and is caught on read. -const VERIFICATION_REUSE_WINDOW: Duration = Duration::from_mins(30); - -/// The network facts the driver needs, kept behind one type so the store itself stays -/// free of any knowledge of routing or commitments. -pub struct MigrationContext { - /// Routing, for close-group rank and possession checks. `None` in devnet and tests. - pub p2p: Option>, - /// This node's peer ID. - pub self_id: Option, - /// This node's address in the key space, for ordering the copy closest-first. - pub self_xor: Option, - /// The responder commitment state, which owns the retention contract. - pub commitment: Option>, - /// Replication settings, for the possession round that gates shedding. - pub replication: Option>, - /// Neighbour-sync state, which the possession challenge needs. - pub sync_state: Option>>, - /// Coordinator for the possession challenges. - pub audit_challenge_coordinator: - Option>, - /// What this node last heard each peer commit to. - /// - /// Used to require that a peer trusted to hold a chunk is currently publishing a - /// claim, rather than sitting between a retired commitment and its next rotation, - /// which is precisely the state a node in the middle of its own migration is in. - pub peer_commitments: Option< - Arc< - tokio::sync::RwLock< - HashMap, - >, - >, - >, - /// Close-group width. - pub close_group_size: usize, -} - -/// How many of the peers auditing this node now have seen its reduced commitment. -/// -/// `received` is who was sent the current root, `current` is the close group as routing -/// sees it at this moment. Only the overlap counts. A peer that received the root and has -/// since left is not going to audit this node, and a peer that has since joined has never -/// seen the root, so neither is evidence that shedding is safe. -fn enough_of_the_group_knows( - received: &HashSet, - current: &[PeerId], - needed: usize, -) -> bool { - if needed == 0 { - return false; - } - current.iter().filter(|p| received.contains(*p)).count() >= needed -} - -impl MigrationContext { - /// How many peers of the close group must have seen the reduced commitment. - /// - /// The same tolerance the pruner applies to possession proofs: all of them for a group - /// of one or two, one short of the group otherwise, so a single unreachable peer - /// cannot veto the migration forever without accepting an uninformed close group. - #[must_use] - pub fn commitment_recipients_needed(&self) -> usize { - prune_proofs_needed(self.close_group_size.saturating_sub(1)) - } - - /// Have enough of this node's close group actually received its reduced commitment? - /// - /// A rotation is not the same as neighbours knowing. Until they have seen the smaller - /// key set they keep auditing against the one this node used to hold, so giving a - /// chunk up before then turns a legitimate migration into a wave of audit failures. - pub async fn neighbours_know_the_commitment(&self) -> bool { - let needed = self.commitment_recipients_needed(); - if needed == 0 { - return false; - } - let Some(state) = self.commitment.as_ref() else { - return false; - }; - let received = state.current_delivered_peers(); - if received.is_empty() { - return false; - } - // Counted against the group as it stands now, not as it stood when the root went - // out. A peer that has since left knowing this node's reduced commitment says - // nothing about the peers that will actually audit it, and letting a departed - // peer satisfy the gate is how a node gives chunks up while its real neighbours - // still hold it to the larger key set. - let Some(current) = self.current_close_group().await else { - return false; - }; - enough_of_the_group_knows(&received, ¤t, needed) - } - - /// This node's close group as routing sees it now, or `None` if the view is too thin - /// to be evidence about a group at all. - async fn current_close_group(&self) -> Option> { - let (Some(p2p), Some(me), Some(self_xor)) = ( - self.p2p.as_ref(), - self.self_id.as_ref(), - self.self_xor.as_ref(), - ) else { - return None; - }; - // Self-inclusive, then self filtered out. The self-excluding call would return - // `close_group_size` *remote* peers, one more than the group actually has, and the - // threshold is computed from a group that includes this node. Four real - // neighbours plus one peer outside the group would then clear a bar meant to - // require five real ones. - let closest = p2p - .dht_manager() - .find_closest_nodes_local_with_self(self_xor, self.close_group_size) - .await; - let peers: Vec = closest - .iter() - .map(|n| n.peer_id) - .filter(|p| p != me) - .collect(); - if peers.len() + 1 < self.close_group_size { - return None; - } - Some(peers) - } - - /// Is this key still answerable under a retained commitment slot? - /// - /// This is the pruner's existing veto, reused verbatim: a key the node could still - /// be challenged on must not lose its last local copy. - #[must_use] - pub fn still_answerable(&self, key: &XorName) -> bool { - self.commitment - .as_ref() - .is_some_and(|state| state.is_held(key)) - } - - /// The width this node measures ranks against: the admission group, not the close - /// group. - #[must_use] - pub fn shed_width(&self) -> usize { - storage_admission_width(self.close_group_size) - } - - /// This node's position in `key`'s admission group. - pub async fn close_group_rank(&self, key: &XorName) -> GroupRank { - let (Some(p2p), Some(me)) = (self.p2p.as_ref(), self.self_id.as_ref()) else { - return GroupRank::Unknown; - }; - let closest = p2p - .dht_manager() - .find_closest_nodes_local_with_self(key, self.shed_width()) - .await; - closest - .iter() - .position(|n| n.peer_id == *me) - .map_or(GroupRank::Outside, GroupRank::Inside) - } - - /// May this node give up `key` without risking its last replica? - /// - /// Only if it is outside the admission group entirely, or sits in that group's last - /// [`SHEDDABLE_TAIL_RANKS`] positions. Never when the answer is unknown. - pub async fn may_shed(&self, key: &XorName) -> bool { - rank_is_sheddable(self.close_group_rank(key).await, self.shed_width()) - } -} - -/// Where this node sits in a key's admission group. -/// -/// `Unknown` is deliberately distinct from `Outside`. Collapsing the two would turn "this -/// node has no routing table to consult" into "no other node is closer", which is a -/// licence to give up every chunk on no evidence whatsoever. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum GroupRank { - /// This node is at this position, counting from the closest. - Inside(usize), - /// This node is not among the closest for this key. - Outside, - /// Routing state is unavailable, so the question cannot be answered. - Unknown, -} - -/// Which of `keys` this node has no proof anyone else is holding. -/// -/// This is the gate on giving a chunk up at all, and it is deliberately the **same** -/// evidence the pruner demands before it deletes: cryptographic possession proofs from -/// all but one of the key's current close group, which is six of seven at -/// production width. -/// -/// Rank alone was not enough. Being far from a chunk says something about who *should* -/// hold it, not about who *does*, and during a fleet-wide migration the nodes that should -/// hold it are exactly the ones that may also be short of space. Nor is the cheap -/// `VerificationRequest` enough: it carries a self-reported `present: bool`, and a node -/// that has silently lost a chunk still answers yes. The challenge here makes a peer -/// return a digest over a nonce it has never seen, which it cannot do without the bytes. -/// -/// Returns the keys that failed, so the caller can name them. An empty result means every -/// key asked about is proven to live somewhere else. -/// -/// Without routing state, every key is unconfirmed: no view of the network is no evidence. -pub async fn unconfirmed_by_neighbours( - store: &Arc, - context: &MigrationContext, - keys: &[XorName], -) -> Vec { - let (Some(p2p), Some(self_id), Some(config), Some(sync_state), Some(coordinator)) = ( - context.p2p.as_ref(), - context.self_id.as_ref(), - context.replication.as_ref(), - context.sync_state.as_ref(), - context.audit_challenge_coordinator.as_ref(), - ) else { - return keys.to_vec(); - }; - - let local_key_count = - usize::try_from(store.current_chunks().unwrap_or(0)).unwrap_or(usize::MAX); - let dht = p2p.dht_manager(); - let mut unconfirmed = Vec::new(); - - for batch in keys.chunks(POSSESSION_BATCH_KEYS) { - // Ask only the peers that are currently closest to each key. A proof from a peer - // that has since moved out of the group is not evidence the chunk will stay there. - let mut targets_by_key: HashMap> = HashMap::new(); - let mut keys_by_peer: HashMap> = HashMap::new(); - for key in batch { - // Self-inclusive, matching the pruner, whose evidence this is. The - // self-excluding call returns one peer more than the key's group holds, and a - // proof from a peer outside it is not evidence the chunk stays in it. - let closest = dht - .find_closest_nodes_local_with_self(key, config.close_group_size) - .await; - let peers: Vec = closest - .iter() - .map(|n| n.peer_id) - .filter(|p| p != self_id) - .collect(); - for peer in &peers { - keys_by_peer.entry(*peer).or_default().push(*key); - } - targets_by_key.insert(*key, peers); - } - - let proofs = prove_peers_hold_records( - &keys_by_peer, - local_key_count, - store, - p2p, - config, - sync_state, - coordinator, - ) - .await; - - // A proof is necessary but not sufficient. The peer must also be currently - // publishing a commitment, so a node that has retired its own and not yet rotated - // a replacement, which is what a node mid-migration looks like, is not counted as - // the reason this node may give a chunk up. - let publishing = peers_publishing_a_recent_commitment(context).await; - - for key in batch { - let group = targets_by_key.get(key).map_or(&[][..], Vec::as_slice); - - // The lookup is self-inclusive, and this node is giving the chunk up, so a - // full group is `close_group_size` peers none of which is this node. Fewer - // than that is a routing view too thin to be evidence about the group at all, - // which is what a table looks like shortly after a restart. This node still - // appearing in the group means it is not outside it after all, and the - // decision to give the chunk up was taken against a view that has since - // changed. - if group.len() < config.close_group_size { - unconfirmed.push(*key); - continue; - } - - // The threshold comes from the configured group size, never from whichever - // subset happens to qualify, nor from however many peers routing returned. - // Deriving it from the filtered list is how two last holders destroy a chunk - // between them: each sees only the other publishing, so each needs exactly one - // proof, each gets it from the other, and both delete. Deriving it from the - // observed length is the same mistake more quietly: a view that has lost a - // peer lowers the bar exactly when it should not be trusted. - let needed = prune_proofs_needed(config.close_group_size); - let qualifying: Vec = group - .iter() - .filter(|p| publishing.contains(*p)) - .copied() - .collect(); - if !target_peers_reported_present(key, &qualifying, &proofs, needed) { - unconfirmed.push(*key); - } - } - } - unconfirmed -} - -/// The peers this node has heard a commitment from recently enough to trust as holders. -async fn peers_publishing_a_recent_commitment(context: &MigrationContext) -> HashSet { - let Some(records) = context.peer_commitments.as_ref() else { - return HashSet::new(); - }; - records - .read() - .await - .iter() - .filter(|(_, record)| { - record.last_commitment().is_some() - && record.received_at.elapsed() < COMMITMENT_FRESHNESS - }) - .map(|(peer, _)| *peer) - .collect() -} - -/// Whether a position in the admission group may be given up. -/// -/// Split out from the routing lookup so the rule itself is testable without a network. -/// `width` is [`storage_admission_width`], not the close-group size: a key this node is -/// outside the admission group for is one the pruner would delete anyway, and inside it -/// only the last [`SHEDDABLE_TAIL_RANKS`] positions may go. -#[must_use] -pub fn rank_is_sheddable(rank: GroupRank, width: usize) -> bool { - // A group no wider than the tail has no tail to give up. Saturating alone would set - // the threshold to zero and make every member sheddable, which is the opposite of - // what a narrow group needs. - let protected_below = if width <= SHEDDABLE_TAIL_RANKS { - width - } else { - width - SHEDDABLE_TAIL_RANKS - }; - match rank { - // No routing to ask. Never a licence: a node with no view of the network has no - // grounds at all for believing anyone else holds the chunk. - GroupRank::Unknown => false, - GroupRank::Outside => true, - GroupRank::Inside(rank) => rank >= protected_below, - } -} - -/// Whether this store needs a migration driver at all. -/// -/// The single predicate both the spawn site and its test use, so "should this node be -/// migrating" cannot be answered one way by the wiring and another way by what checks it. -#[must_use] -pub fn should_migrate(store: &Arc) -> bool { - // Or has a removal to finish, or has something at the environment's path it could not - // open. A node whose retirement was interrupted has no handle and nothing left to - // copy, but its disk has not come back. A node whose environment is a link to storage - // that was not mounted at startup has neither, and its chunks come back when the - // storage does; without a driver it would stay blind to them until a restart. - store.has_legacy() || store.has_cleanup_pending() || store.legacy_dir_is_on_disk() -} - -/// Runs the migration to completion, then returns. -/// -/// Everything it does is idempotent and derived from the filesystem, so a crash at any -/// point costs at most the work of one tick. -pub async fn run(store: Arc, context: MigrationContext, shutdown: CancellationToken) { - let config = store.migration_config().clone(); - if !worth_starting(&store, &config) { - return; - } - - let tick = Duration::from_secs(config.tick_secs.max(1)); - let mut volume_lock: Option = None; - let mut held = LockHold::default(); - let mut next_shed_evaluation = Instant::now(); - let mut next_handle_recovery = Instant::now(); - // A clean verification is a full re-read of everything both stores hold. If - // retirement is then deferred (a read still holds the legacy handle), re-hashing on - // every tick would be minutes of disk for nothing, so a recent pass is reused. - let mut verified: Option<(VerifyReport, Instant)> = None; - - loop { - tokio::select! { - () = shutdown.cancelled() => { - debug!("Storage migration stopping for shutdown"); - return; - } - () = tokio::time::sleep(tick) => {} - } - - // Never hold the volume against the rest of the machine for longer than this, - // whatever the node is waiting on. Dropping it costs a tick: if nobody else wants - // it, the branches below take it straight back. - if held.has_overstayed() { - debug!( - "Held the volume migration lock for {} hour(s); giving it back so any \ - other node on this volume gets a turn", - MAX_VOLUME_LOCK_HOLD.as_secs() / 3600 - ); - volume_lock = None; - held.give_up(); - } - - // Cleanup first. It can put an unmarked directory back under the live name, and - // recovery is what gives the node a handle to it; the other order leaves that - // until the next tick, and the completion check in between would see no handle - // and no pending cleanup and call the migration finished. - let cleanup = cleanup_state(&store); - // Before anything reads the key set: a write that nobody waited for leaves a - // note behind, and only the disk can say what became of it. - store.reconcile_pending_writes().await; - maybe_recover_lost_handle(&store, &mut next_handle_recovery).await; - if cleanup == CleanupState::Finished && !store.has_legacy() { - info!("Storage migration finished; nothing left on disk to clean up"); - return; - } - - match store.migration_phase() { - // Nothing left to migrate. Whether there is anything left to clean up is - // decided at the top of the loop, which is also where this returns from. - MigrationPhase::FilesOnly => { - volume_lock = None; - held.released(); - } - MigrationPhase::Bridging => { - // Held from the first copy through retirement, not released in between: - // a node that let go after copying would let its eleven neighbours start - // theirs before it had returned a byte, which is the exact pile-up the - // lock exists to prevent. The one exception is a node that has become - // permanently stuck (see below), which must not go on excluding the - // others for a release. - if matches!( - take_volume_lock( - &mut volume_lock, - &mut held, - store.root_dir(), - config.lock_dir.as_deref(), - ), - LockStep::WaitATick - ) { - continue; - } - if bridge_tick( - &store, - &context, - &config, - &mut next_shed_evaluation, - &shutdown, - ) - .await - { - // The copier ran. That is the volume lock being used rather than held. - held.note_disk_work(); - } else { - // Copying is blocked on something only an operator can change, so - // stop holding the volume lock against the other nodes here. - volume_lock = None; - held.released(); - } - } - MigrationPhase::Committed => { - // A node that restarted in this phase has no lock, and the work below - // (copying anything that must be kept, then re-reading the whole store to - // verify it) is exactly the disk-heavy work the lock exists to serialise. - if matches!( - take_volume_lock( - &mut volume_lock, - &mut held, - store.root_dir(), - config.lock_dir.as_deref(), - ), - LockStep::WaitATick - ) { - continue; - } - match retire_tick(&store, &context, &config, &mut verified, &shutdown).await { - // Not a return. The environment is gone from the node's point of - // view, but its directory is still being deleted in the background, - // and if that fails there has to be something left to try again. The - // loop exits at the top once nothing is pending. - RetireOutcome::Done => { - volume_lock = None; - held.released(); - } - // Time spent reading or copying is the volume lock doing its job, not - // a node sitting on it. The cap is there for a node that waits, and - // restarting a full verification because a large store took longer - // than the cap would be the cap causing the problem it prevents. - RetireOutcome::Working => held.note_disk_work(), - RetireOutcome::Waiting => {} - RetireOutcome::NoWorkToSerialise => { - // Nothing this node can do will return space, so holding the - // volume lock only stops its neighbours from trying. In R1, where - // retirement is switched off entirely, holding it would mean one - // node per volume copies and the other eleven do nothing for the - // whole release. - volume_lock = None; - held.released(); - } - } - } - } - } -} - -/// Should the driver run at all, and say why in the log if not? -fn worth_starting(store: &Arc, config: &MigrationConfig) -> bool { - if !config.enabled { - warn!( - "Storage migration is disabled. This node will keep reading both stores and \ - will never return the legacy environment's disk space." - ); - return false; - } - // The same question the spawn site asks. A different one here means a driver that is - // started and then returns immediately, which is how a node whose environment is a - // link to storage that was not mounted yet ends up never picking it up. - if !should_migrate(store) { - debug!("No legacy chunk environment; nothing to migrate"); - return false; - } - - let to_copy = store.legacy_only_keys().len(); - info!( - migration_event = "start", - to_copy, - legacy_bytes = store.legacy_bytes(), - "Storage migration starting: {to_copy} chunk(s) still only in the legacy \ - environment, {:.2} GiB to reclaim", - bytes_to_gib(store.legacy_bytes()) - ); - true -} - -/// Retry any removal that did not finish, and say whether the driver is done. -/// -/// Runs independently of the phase. A removal that could not finish leaves nothing to -/// migrate but a disk that has not come back, and the reasons it failed (a name already -/// taken, a directory that could not be flushed, a scanner holding a handle) are the kind -/// that clear on their own. -fn cleanup_state(store: &Arc) -> CleanupState { - if store.has_cleanup_pending() { - store.retry_cleanup(); - } - if store.has_cleanup_pending() || store.legacy_dir_is_on_disk() { - return CleanupState::Pending; - } - CleanupState::Finished -} - -/// Whether anything is left on disk for the driver to see through. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum CleanupState { - /// Something is still there. - Pending, - /// Nothing is. - Finished, -} - -/// Put a lost legacy handle back, if there is one to put back. -/// -/// A node that lost its handle to an environment still on disk cannot read the chunks that -/// live only there. The cause is usually transient, so this runs every tick rather than -/// leaving the node half-blind until somebody restarts it. -async fn maybe_recover_lost_handle(store: &Arc, next_attempt: &mut Instant) { - if Instant::now() < *next_attempt { - return; - } - if store.recover_lost_legacy_handle().await { - info!("Reopened the legacy chunk environment; the migration continues"); - *next_attempt = Instant::now(); - return; - } - // Backed off, because each attempt scans the whole environment and a cause that has - // not cleared in half a minute is unlikely to clear in the next. - *next_attempt = Instant::now() + HANDLE_RECOVERY_INTERVAL; -} - -/// How long this node has had the volume migration lock, and when it may ask again. -/// -/// Split out because the rule is easy to get wrong in one branch and not another: a -/// branch that takes the lock without recording when, or that gives it up without a -/// cooldown, silently opts out of the cap that stops one node holding a whole machine. -#[derive(Default)] -struct LockHold { - /// When the lock was taken, or when it was last used for real disk work. - since: Option, - /// Before this, do not ask for it again. - cooldown_until: Option, -} - -impl LockHold { - /// Record that the lock has just been taken. - fn taken(&mut self) { - self.since = Some(Instant::now()); - } - - /// Record that it is no longer held. - fn released(&mut self) { - self.since = None; - } - - /// Record that this tick used the lock for what it is for. - /// - /// Copying and verifying are the exclusive disk work the lock exists to serialise, so - /// time spent on them is not time spent sitting on it. Without this a node with a - /// large store would have the cap fire in the middle of a verification pass and - /// restart it, which is the cap causing the problem it prevents. - fn note_disk_work(&mut self) { - self.since = Some(Instant::now()); - } - - /// Has this node held the lock past the cap without using it? - fn has_overstayed(&self) -> bool { - self.since - .is_some_and(|at| at.elapsed() >= MAX_VOLUME_LOCK_HOLD) - } - - /// Give the lock up and stand back so somebody else can take it. - fn give_up(&mut self) { - self.since = None; - self.cooldown_until = Some(Instant::now() + VOLUME_LOCK_COOLDOWN); - } - - /// May this node ask for the lock yet? - fn may_ask(&self) -> bool { - self.cooldown_until - .is_none_or(|until| Instant::now() >= until) - } -} - -/// What taking the volume lock produced for the driver loop. -enum LockStep { - /// Held, or not needed because none is possible here. - Proceed, - /// Someone else has it. Try again next tick. - WaitATick, -} - -/// Take the volume lock if it is not already held, stamping when it was taken. -/// -/// One place, because the stamp is what stops a node holding the volume against every -/// other node on the machine, and a branch that acquires without stamping silently opts -/// out of that. -fn take_volume_lock( - lock: &mut Option, - held: &mut LockHold, - root_dir: &Path, - scope: Option<&Path>, -) -> LockStep { - if lock.is_some() { - return LockStep::Proceed; - } - // After giving the volume up at the cap, stand back for a moment. Reacquiring in the - // same breath would hand nobody anything. - if !held.may_ask() { - return LockStep::WaitATick; - } - match VolumeLock::try_acquire(root_dir, scope) { - LockAttempt::Acquired(taken) => { - *lock = Some(taken); - held.taken(); - LockStep::Proceed - } - LockAttempt::Busy => { - debug!("Another node on this volume is migrating; waiting"); - LockStep::WaitATick - } - // No lock is possible here, so waiting for one would strand this node - // permanently. Proceed; the slack floor is the backstop. - LockAttempt::Unavailable => LockStep::Proceed, - } -} - -/// One pass of the copier. Returns `false` when this node cannot make progress that -/// needs the volume to itself. -async fn bridge_tick( - store: &Arc, - context: &MigrationContext, - config: &MigrationConfig, - next_shed_evaluation: &mut Instant, - shutdown: &CancellationToken, -) -> bool { - let remaining = store.legacy_only_keys(); - if remaining.is_empty() { - if let Err(e) = store.commit_to_files() { - // Not progress, and not something exclusive disk access fixes. Saying it was - // would reset the hold cap every tick and let this node keep the volume from - // every other node on the machine for as long as the failure lasts. - warn!( - "Everything is copied but the migration commitment could not be recorded: \ - {e}. Retrying on the next tick." - ); - return false; - } - return true; - } - - let ordered = rank_closest_first(remaining, context.self_xor); - let batch: Vec = ordered - .into_iter() - .take(config.batch_chunks.max(1)) - .collect(); - let report = match store - .copy_batch( - &batch, - config.copier_slack_bytes(), - config.copier_throttle_mib_per_sec, - shutdown, - ) - .await - { - Ok(report) => report, - Err(e) => { - // Still retried every tick, but the volume lock goes back: if this is - // permanent, holding it would block every other node on the volume on a node - // that is getting nowhere. - warn!("Storage migration copy failed: {e}. Retrying on the next tick."); - return false; - } - }; - - if report.copied > 0 { - debug!( - "Storage migration copied {} chunk(s) ({:.2} GiB) this pass", - report.copied, - bytes_to_gib(report.bytes) - ); - // A migration runs for hours. One periodic line at info level is what an operator - // watching a node actually sees, and what says the copier has not silently stalled. - let left = store.legacy_only_keys().len(); - if left % PROGRESS_LOG_EVERY < usize::try_from(report.copied).unwrap_or(usize::MAX) { - let no_rollback = store.writes_without_a_rollback_copy(); - info!( - migration_event = "progress", - remaining = left, - no_rollback_copy = no_rollback, - "Storage migration: {left} chunk(s) left to copy out of the legacy \ - environment, {no_rollback} write(s) made without a rollback copy" - ); - } - } - if report.unusable > 0 { - warn!( - "{} chunk(s) in the legacy environment did not match their own address and \ - were dropped from the key set", - report.unusable - ); - } - - if report.stopped_for_space { - // Out of space. Whatever happens next, this node is not going to write more until - // something changes, so it stops excluding its neighbours from the volume. That - // covers the 72-hour shed hold as well as an outright refusal: holding the lock - // for three days would leave every other node on the volume unmigrated. - if Instant::now() < *next_shed_evaluation { - return false; - } - *next_shed_evaluation = Instant::now() + SHED_REEVALUATION_INTERVAL; - return evaluate_shed(store, context, config).await; - } - true -} - -/// Decide whether the node may give up what it could not copy. -async fn evaluate_shed( - store: &Arc, - context: &MigrationContext, - config: &MigrationConfig, -) -> bool { - let remaining = store.legacy_only_keys(); - let short_by = remaining.len(); - - // Read from the one switch the auditors read, not from a second copy of it. There - // used to be two constants of the same name with two environment overrides, one on - // each side of this decision, and nothing coupling them: a node could have been - // willing to shed while every peer was still applying the full penalty, which is the - // exact outcome the release ordering exists to prevent. - if !crate::replication::config::close_group_storage_penalty_suspended() { - warn!( - "This node cannot fit {short_by} chunk(s) in the file store, but this release \ - has audit penalties switched back on, so giving anything up now would be \ - penalised by every peer. Keeping both stores. Add disk, or migrate this node \ - on a build that still suspends penalties." - ); - return false; - } - - if !config.allow_shed { - warn!( - "This node cannot fit {short_by} chunk(s) in the file store and shedding is \ - turned off. Add disk, or set storage.migration.allow_shed. Until then it \ - keeps serving from both stores and the legacy environment stays." - ); - return false; - } - - let state = store.migration_state(); - - // Wait for this node's turn. A close group is split into waves so at most - // CONCURRENT_MIGRATIONS_PER_GROUP of it are giving chunks up at once; if all seven - // holders went together, none could prove to the others that a copy survived and the - // whole group would sit deadlocked waiting on each other. Only nodes that have to give - // something up wait: a node with room has already copied everything and retired. - let wave = migration_wave_for(context.self_id.as_ref(), context.close_group_size); - if !wave_has_opened(&state, config, wave) { - info!( - "This node is {short_by} chunk(s) short of disk and is in migration wave {wave} \ - of {}. Its turn opens {} hour(s) after this build first started, so the rest of \ - its close group stays steady and can keep serving what it is about to give up.", - migration_wave_count(context.close_group_size), - config - .shed_hold_hours - .saturating_add(wave.saturating_mul(config.wave_hours)) - ); - return false; - } - - // Kept as its own check even though the wave now starts after it: the hold is about - // peers on an older build still applying the penalty, the wave is about the close - // group being able to cover for whoever moves. Different reasons, both required. - if !state.shed_hold_elapsed(config) { - info!( - "This node is {short_by} chunk(s) short of disk. Holding for {} hour(s) after \ - first start before giving any up, so peers still on an older build have \ - upgraded and stopped penalising a shed.", - config.shed_hold_hours - ); - return false; - } - - // First filter, and the cheap one: a node never gives up a chunk it is near the front - // of the group for. In practice it rarely fires, by construction, because the copier - // walks closest-first, so whatever is left when the disk fills is the far end of the - // list. Finding a protected key still uncopied means the node could not fit even the - // chunks it is closest to, which is exactly when it must not shed anything. - let mut protected = Vec::new(); - for key in &remaining { - if !context.may_shed(key).await { - protected.push(*key); - if protected.len() >= REFUSAL_SAMPLE { - break; - } - } - } - if !protected.is_empty() { - let sample: Vec = protected.iter().map(hex::encode).collect(); - warn!( - "This node is {short_by} chunk(s) short of disk, and at least {} of them are \ - chunks it is near the front of the group for (for example {}). It will not \ - give those up. The legacy environment stays and its disk is not returned \ - until storage is added.", - protected.len(), - sample.join(", ") - ); - return false; - } - - // Second filter, and the one that decides it: proof that somebody else holds every - // chunk this node is about to give up. Being far from a chunk is not evidence - // that a copy exists. During a fleet-wide migration the nodes that ought to hold it - // are exactly the ones that may also be out of disk, so the question has to be asked - // rather than inferred. - info!( - "Checking that other nodes hold the {short_by} chunk(s) this node cannot fit, \ - before giving any of them up" - ); - let unconfirmed = unconfirmed_by_neighbours(store, context, &remaining).await; - if !unconfirmed.is_empty() { - let sample: Vec = unconfirmed - .iter() - .take(REFUSAL_SAMPLE) - .map(hex::encode) - .collect(); - warn!( - "{} of the {short_by} chunk(s) this node cannot fit could not be proven to \ - exist anywhere else (for example {}). Nothing is given up and the legacy \ - environment stays. Add disk, or wait for replication to place them.", - unconfirmed.len(), - sample.join(", ") - ); - return false; - } - - info!( - migration_event = "shed", - shed = short_by, - "Every one of the {short_by} chunk(s) this node cannot fit is proven to be held \ - elsewhere. Committing to what it can hold. They stay readable from the legacy \ - environment until it is removed, and replication refetches whatever still belongs \ - here once there is room." - ); - if let Err(e) = store.commit_to_files() { - warn!("Could not record the migration commitment: {e}"); - return false; - } - true -} - -/// The keys still only in the legacy store that this node is too close to give up. -async fn keys_this_node_must_not_give_up( - store: &Arc, - context: &MigrationContext, -) -> Vec { - let mut must_keep = Vec::new(); - for key in store.legacy_only_keys() { - if !context.may_shed(&key).await { - must_keep.push(key); - } - } - must_keep -} - -/// Re-ask every network gate, after verification and immediately before the deletion. -/// -/// Verification re-reads the whole store and can run for hours. A gate satisfied before it -/// started says nothing about the moment of deletion: peers leave, replicas are pruned -/// elsewhere, and a write whose file half failed adds a fresh legacy-only key that has -/// faced none of these checks. This is the last point at which the answer can still be -/// acted on, so it is the point at which it has to be true. -async fn every_gate_still_holds( - store: &Arc, - context: &MigrationContext, - candidates: &std::collections::BTreeSet, -) -> Option { - // A key that joined the legacy-only set since the snapshot was taken has been through - // none of this. Stop now rather than asking the gates about a set that has already - // moved; the next tick copies it and takes a fresh snapshot. - let live: std::collections::BTreeSet = store.legacy_only_keys().into_iter().collect(); - if live != *candidates { - debug!( - "Legacy environment not retired: the set changed while the gates were being \ - checked ({} keys then, {} now)", - candidates.len(), - live.len() - ); - return Some(RetireOutcome::Waiting); - } - if !keys_this_node_must_not_give_up(store, context) - .await - .is_empty() - { - debug!("Legacy environment not retired: the shed rule changed during verification"); - return Some(RetireOutcome::Waiting); - } - if let Some(outcome) = shedding_is_still_safe(store, context, candidates).await { - return Some(outcome); - } - // And the retention contract once more, for the same reason. - if let Some(reason) = store.retirement_blocker(|k| context.still_answerable(k)) { - debug!("Legacy environment not retired: {reason}"); - return Some(RetireOutcome::Waiting); - } - None -} - -/// The last two questions before anything is deleted, asked in this order because the -/// order is the safety argument: reduce the claim, let the group learn it, then give the -/// chunks up. -/// -/// Returns `Some` with the reason to stop, or `None` when it is safe to proceed. -async fn shedding_is_still_safe( - store: &Arc, - context: &MigrationContext, - shedding: &std::collections::BTreeSet, -) -> Option { - // Nothing below is reached until the node has reduced its commitment (the phase - // is `Committed`) and that reduction has been rebuilt and published. What remains - // is to confirm the close group has actually *received* it, and that the chunks - // being given up still exist elsewhere. Only then is anything deleted. - if !shedding.is_empty() { - // A rotation is not the same as neighbours knowing. Until they have the - // smaller key set they keep auditing this node against the one it used to - // hold, and a wave of audit failures is as damaging as losing the chunks. - // Asked only when there is a commitment to deliver. A node whose file-backed set - // is empty commits to nothing, so there is no hash for a neighbour to acknowledge - // and this gate would never open, stranding its disk for good. Nothing is lost by - // skipping it: the gate exists to stop neighbours auditing this node against a key - // set it no longer holds, and a node claiming nothing cannot fail such an audit. - // The possession check below, which proves every chunk being given up still exists - // elsewhere, is the gate that protects the data, and it still runs. - let commits_to_nothing = store - .committable_keys() - .await - .is_ok_and(|keys| keys.is_empty()); - if commits_to_nothing { - info!( - "This node's file-backed set is empty, so it commits to nothing and has \ - no reduced commitment for its close group to receive. Proceeding to the \ - possession check on the {} chunk(s) it is giving up.", - shedding.len() - ); - } else if !context.neighbours_know_the_commitment().await { - info!( - "Holding: {} of this node's close group must receive its reduced \ - commitment before it gives up {} chunk(s). {} have it so far.", - context.commitment_recipients_needed(), - shedding.len(), - context - .commitment - .as_ref() - .map_or(0, |s| s.current_delivered_peer_count()) - ); - // Give the volume back while waiting on this. It is a network condition, not - // a disk one, and it may never resolve: holding the lock through it would let - // one node stop every other node on the machine from ever starting. - return Some(RetireOutcome::NoWorkToSerialise); - } - // Asked again here, not only when the node committed. Hours pass in between, - // the group moves, and a peer that held a copy then may not now. This is the - // last moment at which the answer still matters. - let ordered: Vec = shedding.iter().copied().collect(); - let unconfirmed = unconfirmed_by_neighbours(store, context, &ordered).await; - if !unconfirmed.is_empty() { - let sample: Vec = unconfirmed - .iter() - .take(REFUSAL_SAMPLE) - .map(hex::encode) - .collect(); - warn!( - "{} of the {} chunk(s) this node is giving up can no longer be proven \ - to exist elsewhere (for example {}). The legacy environment stays.", - unconfirmed.len(), - shedding.len(), - sample.join(", ") - ); - return Some(RetireOutcome::NoWorkToSerialise); - } - } - None -} - -/// What one pass of the retirement gate concluded. -enum RetireOutcome { - /// The legacy environment is gone. The driver is finished. - Done, - /// Exclusive disk work happened this tick: copying, or re-reading the store to verify - /// it. Keep the volume, and count the time as time spent using it rather than time - /// spent holding it. - Working, - /// Still working towards it, but waiting on a clock rather than on the disk. Keep the - /// volume, because the node is about to need it, but let the hold cap run. - Waiting, - /// Blocked on something no amount of exclusive disk access will fix. - NoWorkToSerialise, -} - -/// When this node last said out loud that its migration needs a person. -static LAST_OPERATOR_WARNING: parking_lot::Mutex> = parking_lot::Mutex::new(None); - -/// How often to repeat it. Often enough to be noticed, rarely enough not to drown the log. -const OPERATOR_WARNING_INTERVAL: Duration = Duration::from_hours(1); - -/// Should the "this needs a person" warning be repeated now? -/// -/// The condition it reports is checked on every tick and does not clear on its own, so -/// without this it would be a line every thirty seconds for as long as the node runs. -fn operator_should_hear_again() -> bool { - let mut last = LAST_OPERATOR_WARNING.lock(); - let now = Instant::now(); - if last.is_some_and(|at| now.duration_since(at) < OPERATOR_WARNING_INTERVAL) { - return false; - } - *last = Some(now); - true -} - -/// The retention contract, asked before anything else in the tick. -/// -/// `Some` with what the driver should do, or `None` when nothing is in the way. -fn blocked_before_the_gates( - store: &Arc, - context: &MigrationContext, - config: &MigrationConfig, -) -> Option { - let reason = store.retirement_blocker(|k| context.still_answerable(k))?; - // Three blockers no amount of exclusive disk access will clear: an environment this - // node cannot read, one it cannot classify at all, and one it must not delete because - // it is a link to somewhere else. Each needs a person, so give the volume back to the - // nodes that can use it and say so where an operator will see it rather than at debug. - if store.has_lost_its_legacy_handle() - || store.legacy_cannot_be_classified() - || store.legacy_is_a_link() - { - if operator_should_hear_again() { - warn!( - migration_event = "needs_an_operator", - "The legacy chunk environment will not be retired automatically: {reason}" - ); - } - return Some(RetireOutcome::NoWorkToSerialise); - } - debug!("Legacy environment not retired yet: {reason}"); - Some(if config.retire_legacy { - RetireOutcome::Waiting - } else { - // Retirement is switched off on this node, so it will never free its disk here - // however long it waits. - RetireOutcome::NoWorkToSerialise - }) -} - -/// One pass of the retirement gate. -async fn retire_tick( - store: &Arc, - context: &MigrationContext, - config: &MigrationConfig, - verified: &mut Option<(VerifyReport, Instant)>, - shutdown: &CancellationToken, -) -> RetireOutcome { - if let Some(outcome) = blocked_before_the_gates(store, context, config) { - return outcome; - } - - // Re-check the shed rule against live routing immediately before the destructive - // step, not once when the node committed hours ago. Two things put a key back into - // the legacy-only set after that decision: a file that failed verification and is now - // served from the legacy copy, and a write whose file half failed. Neither went - // through the rank check, and both would be thrown away by the removal below. - let must_keep = keys_this_node_must_not_give_up(store, context).await; - if !must_keep.is_empty() { - warn!( - "{} chunk(s) are still only in the legacy environment and this node is too \ - close to them to give them up. Copying them before anything is removed.", - must_keep.len() - ); - match store - .copy_batch( - &must_keep, - config.copier_slack_bytes(), - config.copier_throttle_mib_per_sec, - shutdown, - ) - .await - { - Ok(report) if report.stopped_for_space => { - warn!( - "Out of disk while copying {} chunk(s) this node must not give up. \ - The legacy environment stays until there is room for them.", - must_keep.len() - ); - *verified = None; - return RetireOutcome::NoWorkToSerialise; - } - Ok(_) => {} - Err(e) => { - warn!("Could not copy the chunks this node must keep: {e}"); - *verified = None; - return RetireOutcome::NoWorkToSerialise; - } - } - // Anything copied changed the file store, so a previous verification no longer - // covers it. - *verified = None; - return RetireOutcome::Working; - } - - // The real report from a recent pass, never a fabricated one. Reuse deliberately does - // NOT refresh the window: re-arming it from a reused proof would let a node that - // keeps deferring retirement run the verification exactly once and coast on it. - let reusable = verified - .filter(|(_, at)| at.elapsed() < VERIFICATION_REUSE_WINDOW) - .map(|(proof, _)| proof); - let proof = match reusable { - Some(proof) => proof, - None => match store - .verify_before_retire(config.copier_throttle_mib_per_sec, shutdown) - .await - { - Ok(proof) => { - if proof.is_clean() { - *verified = Some((proof, Instant::now())); - } - proof - } - Err(e) => { - // A pass that failed is not progress, however quickly it failed, and - // treating it as work would let a node whose store cannot be read hold - // the volume against every other node on the machine for good. - warn!("Pre-retirement verification failed: {e}. Retrying on the next tick."); - return RetireOutcome::NoWorkToSerialise; - } - }, - }; - if !proof.is_clean() { - *verified = None; - warn!( - "Pre-retirement verification found {} chunk(s) that are damaged in the file \ - store and cannot be repaired from the legacy environment. The legacy \ - environment stays.", - proof.unrepairable() - ); - return RetireOutcome::NoWorkToSerialise; - } - - // Snapshotted BEFORE the gates, not after. Every gate below is asked about exactly - // this set, and exactly this set is what the removal is permitted to destroy. Taken - // afterwards, a key that joined between the last gate and the snapshot would be - // counted as approved having passed nothing, which is the case the gates exist for. - let approved: std::collections::BTreeSet = - store.legacy_only_keys().into_iter().collect(); - - if let Some(outcome) = every_gate_still_holds(store, context, &approved).await { - return outcome; - } - - let kept = store.current_chunks().unwrap_or(0); - let shed = store.migration_state().shed_key_count; - match store - .retire_legacy( - &proof, - &|k: &XorName| context.still_answerable(k), - &approved, - ) - .await - { - Ok(freed) => { - log_migration_complete(kept, shed, freed); - RetireOutcome::Done - } - Err(e) => { - debug!("Legacy environment not retired yet: {e}"); - RetireOutcome::Waiting - } - } -} - -/// Convert a byte count to GiB for human-readable log messages. -#[allow(clippy::cast_precision_loss)] // display only -#[cfg_attr(not(feature = "logging"), allow(dead_code))] -fn bytes_to_gib(bytes: u64) -> f64 { - bytes as f64 / (1024.0 * 1024.0 * 1024.0) -} - -#[cfg(test)] -#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] -mod tests { - use super::*; - use serial_test::serial; - use tempfile::TempDir; - - fn peer_id(byte: u8) -> PeerId { - let mut bytes = [0u8; 32]; - if let Some(slot) = bytes.first_mut() { - *slot = byte; - } - PeerId::from_bytes(bytes) - } - - /// The width shedding is measured against: the admission group, not the close group. - const WIDTH: usize = storage_admission_width(7); - - #[test] - fn shedding_is_measured_against_the_width_the_pruner_protects() { - // The pruner treats the admission group (close group plus its margin) as strictly - // in-range and refuses to delete inside it. A one-off migration must not be more - // willing to drop a chunk than the thing that runs every day. - assert_eq!(WIDTH, storage_admission_width(7)); - assert!( - storage_admission_width(7) > 7, - "the admission group is wider than the close group" - ); - // A rank the close group would have called sheddable is protected here. - assert!(!rank_is_sheddable(GroupRank::Inside(5), WIDTH)); - assert!(!rank_is_sheddable(GroupRank::Inside(6), WIDTH)); - } - - #[test] - fn a_node_never_gives_up_a_chunk_it_is_among_the_closest_to() { - // This node is one of the closest for these ranks. Giving one of them up is the - // case where every short-of-space holder could drop the same chunk and take its - // last replica, so it is refused outright. - for rank in 0..WIDTH - SHEDDABLE_TAIL_RANKS { - assert!( - !rank_is_sheddable(GroupRank::Inside(rank), WIDTH), - "rank {rank} must be protected" - ); - } - // The last two positions may be given up: a chunk has exactly one holder at each, - // so it is only ever a candidate for two of its holders. - for rank in WIDTH - SHEDDABLE_TAIL_RANKS..WIDTH { - assert!( - rank_is_sheddable(GroupRank::Inside(rank), WIDTH), - "rank {rank} is in the tail and may be shed" - ); - } - // Out of range entirely: nothing to protect. - assert!(rank_is_sheddable(GroupRank::Outside, WIDTH)); - // No routing to consult is never a licence. - assert!(!rank_is_sheddable(GroupRank::Unknown, WIDTH)); - } - - #[test] - fn a_group_narrower_than_the_tail_protects_everything_in_it() { - // A group with no tail has nothing to give up. Subtracting saturatingly would put - // the threshold at zero and make every member sheddable, which is exactly - // backwards for the narrowest groups. - assert!(!rank_is_sheddable(GroupRank::Inside(0), 2)); - assert!(!rank_is_sheddable(GroupRank::Inside(0), 1)); - assert!(!rank_is_sheddable(GroupRank::Inside(1), 2)); - // Out of the group entirely is still out. - assert!(rank_is_sheddable(GroupRank::Outside, 2)); - // And a group with a tail still has one. - assert!(!rank_is_sheddable(GroupRank::Inside(0), 3)); - assert!(rank_is_sheddable(GroupRank::Inside(1), 3)); - } - - #[test] - fn the_marker_round_trips_and_a_corrupt_one_starts_over_conservatively() { - let dir = TempDir::new().expect("temp dir"); - let mut state = MigrationState::new(MigrationPhase::Bridging); - state.phase = MigrationPhase::Committed; - state.shed_key_count = 12; - state.committed_at_unix = Some(1_700_000_000); - state.save(dir.path()).expect("save"); - - let loaded = MigrationState::load_or_new(dir.path(), MigrationPhase::Bridging); - assert_eq!(loaded.phase, MigrationPhase::Committed); - assert_eq!(loaded.shed_key_count, 12); - assert_eq!(loaded.committed_at_unix, Some(1_700_000_000)); - - // An unreadable marker restarts the clocks rather than being fatal. Losing it - // delays a migration and can never rush one. - std::fs::write(state_path(dir.path()), b"not json").expect("corrupt"); - let recovered = MigrationState::load_or_new(dir.path(), MigrationPhase::Bridging); - assert_eq!(recovered.phase, MigrationPhase::Bridging); - assert_eq!(recovered.committed_at_unix, None); - } - - #[test] - fn the_shed_hold_and_retirement_clocks_run_from_recorded_times() { - let config = MigrationConfig { - shed_hold_hours: 72, - retire_delay_hours: MIN_RETIRE_DELAY_HOURS, - ..MigrationConfig::default() - }; - - let mut state = MigrationState::new(MigrationPhase::Bridging); - assert!(!state.shed_hold_elapsed(&config), "just started"); - state.first_start_unix = now_unix().saturating_sub(73 * 3600); - assert!(state.shed_hold_elapsed(&config)); - - assert!( - !state.retire_delay_elapsed(&config), - "never committed, so the clock has not started" - ); - state.committed_at_unix = Some(now_unix().saturating_sub(3600)); - assert!(!state.retire_delay_elapsed(&config), "an hour is not four"); - state.committed_at_unix = - Some(now_unix().saturating_sub(MIN_RETIRE_DELAY_HOURS * 3600 + 60)); - assert!(state.retire_delay_elapsed(&config)); - } - - #[test] - fn only_one_node_on_a_volume_migrates_at_a_time() { - let volume = TempDir::new().expect("temp dir"); - let node_a = volume.path().join("node-a"); - let node_b = volume.path().join("node-b"); - std::fs::create_dir_all(&node_a).expect("mkdir"); - std::fs::create_dir_all(&node_b).expect("mkdir"); - - let LockAttempt::Acquired(held) = VolumeLock::try_acquire(&node_a, None) else { - panic!("the first node must take the lock"); - }; - assert!( - matches!(VolumeLock::try_acquire(&node_b, None), LockAttempt::Busy), - "a second node on the same volume must be told to wait, not that no lock exists" - ); - - drop(held); - assert!( - matches!( - VolumeLock::try_acquire(&node_b, None), - LockAttempt::Acquired(_) - ), - "and take it once the first is done" - ); - } - - /// Seed a real LMDB chunk store, the way a node upgrading into this build has one. - async fn seed_legacy(root: &std::path::Path, count: u32) -> Vec { - let lmdb = crate::storage::LmdbStorage::new(crate::storage::LmdbStorageConfig { - root_dir: root.to_path_buf(), - verify_on_read: true, - max_map_size: 0, - disk_reserve: 0, - }) - .await - .expect("open legacy"); - let mut keys = Vec::new(); - for i in 0..count { - let content = format!("legacy-chunk-{i}").into_bytes(); - let addr = crate::client::compute_address(&content); - lmdb.put(&addr, &content).await.expect("legacy put"); - keys.push(addr); - } - lmdb.wait_idle().await; - drop(lmdb); - keys - } - - /// The whole point, end to end: a node that starts with an LMDB chunk store and a - /// disk to hold it finishes with the chunks in files and the LMDB gone. - /// - /// Driven by `run`, the same entry point node startup calls, rather than by poking the - /// pieces. That matters: the wiring that calls it went missing once and every test - /// passed, because they all built the store directly and a node with no legacy store - /// starts no migration. - #[tokio::test] - async fn a_node_with_room_copies_everything_and_removes_the_legacy_store() { - const CHUNKS: u32 = 24; - - let tmp = TempDir::new().expect("temp dir"); - // Nested, so the volume lock this node takes lives in its own directory rather - // than one shared with every other test running in parallel. - let root = tmp.path().join("node"); - std::fs::create_dir_all(&root).expect("mkdir"); - let keys = seed_legacy(&root, CHUNKS).await; - - let mut config = crate::storage::ChunkStoreConfig { - root_dir: root.clone(), - ..crate::storage::ChunkStoreConfig::test_default() - }; - // Deliberately NOT setting `retire_legacy`. The shipped default has to be what - // carries this all the way to a removed environment, or the release migrates every - // node and reclaims nothing. - config.migration.tick_secs = 1; - // Scoped to this test's own directory. In production the lock is keyed by the - // filesystem, so without this every test on this machine would serialise against - // every other one that runs a migration. - config.migration.lock_dir = Some(root.clone()); - config.migration.copier_throttle_mib_per_sec = 0; - let store = Arc::new( - crate::storage::ChunkStore::new(config) - .await - .expect("open store"), - ); - - // Precondition: everything is in the legacy store and nothing is in files. - assert!(store.has_legacy(), "the node must start with an LMDB store"); - assert_eq!(store.migration_phase(), MigrationPhase::Bridging); - assert_eq!(store.legacy_only_keys().len(), CHUNKS as usize); - assert!(root.join("chunks.mdb").exists()); - - let shutdown = CancellationToken::new(); - let driver = tokio::spawn(run( - Arc::clone(&store), - MigrationContext { - p2p: None, - self_id: None, - self_xor: None, - commitment: None, - replication: None, - sync_state: None, - audit_challenge_coordinator: None, - peer_commitments: None, - close_group_size: 7, - }, - shutdown.clone(), - )); - - // The copier runs on its own and settles once nothing is left only in the legacy - // store. This node has room, so it sheds nothing and needs no network at all. - wait_for( - &store, - MigrationPhase::Committed, - "the copier should finish", - ) - .await; - assert!( - store.legacy_only_keys().is_empty(), - "every chunk should have been copied" - ); - - // Stand in for the commitment builder, which lives in the replication engine: the - // retirement gate wants the reduced commitment published and its window elapsed. - store.note_commitment_rebuilt(); - store.note_commitment_rebuilt(); - store.force_migration_state(|s| { - s.committed_at_unix = - Some(now_unix().saturating_sub(MIN_RETIRE_DELAY_HOURS * 3600 + 60)); - }); - - wait_for( - &store, - MigrationPhase::FilesOnly, - "retirement should complete", - ) - .await; - shutdown.cancel(); - let _ = tokio::time::timeout(Duration::from_secs(10), driver).await; - - // The point of the whole exercise: the LMDB is gone from the filesystem. - assert!( - !root.join("chunks.mdb").exists(), - "the legacy store must be removed, which is the only moment disk comes back" - ); - assert!(!store.has_legacy()); - - // And nothing was lost: every chunk still reads, now out of a file. - assert_eq!(store.current_chunks().expect("count"), u64::from(CHUNKS)); - for (i, key) in keys.iter().enumerate() { - let expected = format!("legacy-chunk-{i}").into_bytes(); - assert_eq!( - store.get(key).await.expect("get").expect("present"), - expected, - "chunk {i} did not survive the migration" - ); - } - - // In files, under the suffix shard its address names. - let sample = keys.first().copied().expect("a key"); - let path = root - .join(crate::storage::file_store::CHUNKS_DIR_NAME) - .join(format!("{:02x}", sample.last().copied().unwrap_or(0))) - .join(hex::encode(sample)); - assert!(path.exists(), "expected a chunk file at {}", path.display()); - } - - /// The other half: a node that cannot fit its chunks and cannot prove anyone else - /// holds them keeps both stores and deletes nothing. - /// - /// This is the case that must fail safe. The node is out of disk, so it would like to - /// give chunks up, but with no view of the network it cannot show a single one exists - /// elsewhere. Refusing costs it disk. Proceeding would cost the network data. - #[tokio::test] - async fn a_node_that_cannot_prove_its_chunks_are_safe_deletes_nothing() { - const CHUNKS: u32 = 8; - - let tmp = TempDir::new().expect("temp dir"); - let root = tmp.path().join("node"); - std::fs::create_dir_all(&root).expect("mkdir"); - let keys = seed_legacy(&root, CHUNKS).await; - - let mut config = crate::storage::ChunkStoreConfig { - root_dir: root.clone(), - // Nothing will fit: the copier stops for space on its first chunk. - disk_reserve: u64::MAX / 2, - ..crate::storage::ChunkStoreConfig::test_default() - }; - config.migration.retire_legacy = true; - config.migration.tick_secs = 1; - // Scoped to this test's own directory. In production the lock is keyed by the - // filesystem, so without this every test on this machine would serialise against - // every other one that runs a migration. - config.migration.lock_dir = Some(root.clone()); - // Elapsed, so the hold is not what is doing the refusing here. - config.migration.shed_hold_hours = 0; - config.migration.wave_hours = 0; - let store = Arc::new( - crate::storage::ChunkStore::new(config) - .await - .expect("open store"), - ); - assert_eq!(store.legacy_only_keys().len(), CHUNKS as usize); - - let shutdown = CancellationToken::new(); - let driver = tokio::spawn(run( - Arc::clone(&store), - MigrationContext { - p2p: None, - self_id: None, - self_xor: None, - commitment: None, - replication: None, - sync_state: None, - audit_challenge_coordinator: None, - peer_commitments: None, - close_group_size: 7, - }, - shutdown.clone(), - )); - - // Give it long enough to have tried, re-tried, and evaluated shedding. - tokio::time::sleep(Duration::from_secs(5)).await; - shutdown.cancel(); - let _ = tokio::time::timeout(Duration::from_secs(10), driver).await; - - assert_eq!( - store.migration_phase(), - MigrationPhase::Bridging, - "a node that cannot prove its chunks are held elsewhere must not commit" - ); - assert!( - root.join("chunks.mdb").exists(), - "and must not remove the only copy of them" - ); - assert_eq!(store.legacy_only_keys().len(), CHUNKS as usize); - for (i, key) in keys.iter().enumerate() { - assert_eq!( - store.get(key).await.expect("get").expect("present"), - format!("legacy-chunk-{i}").into_bytes(), - "chunk {i} must still be served throughout" - ); - } - } - - /// Poll until the store reaches `phase`, or fail with what it reached instead. - /// - /// The deadline is generous because it is measured on the wall clock while the driver - /// it is waiting on runs on the runtime. On a saturated machine both stretch, and a - /// deadline sized for the work rather than for the contention turns a slow build into - /// a failing test. The work itself is two ticks. - async fn wait_for(store: &Arc, phase: MigrationPhase, what: &str) { - let deadline = std::time::Instant::now() + Duration::from_mins(3); - while std::time::Instant::now() < deadline { - if store.migration_phase() == phase { - return; - } - tokio::time::sleep(Duration::from_millis(100)).await; - } - panic!( - "{what}: still in {:?} after the deadline, expected {phase:?}", - store.migration_phase() - ); - } - - /// A marker that cannot be used is replaced on disk, not just in memory. - /// - /// The shed hold counts from `first_start_unix`, and the whole reason this is written - /// at startup rather than at the first phase change is that a marker written later - /// would restart that clock on every reboot. A marker that is present but unusable used - /// to defeat that: it was replaced in memory and left on disk, so every start read the - /// same bad file and stamped a fresh clock. A node restarting more often than the hold - /// would then never become eligible to shed and never finish migrating, which is the - /// failure the doc comment on `load_or_create` names. - #[test] - fn an_unusable_marker_is_replaced_on_disk_so_the_hold_does_not_restart() { - for (name, bad) in [ - ("truncated", br#"{"schema":1,"phase":"brid"#.to_vec()), - ("not json at all", b"\x00\x01\x02".to_vec()), - ( - "a newer schema", - serde_json::json!({ - "schema": 9_999, - "phase": "bridging", - "first_start_unix": 1, - "committed_at_unix": null, - "rebuilds_since_commit": 0, - "shed_key_count": 0, - "kept_key_count": 0, - }) - .to_string() - .into_bytes(), - ), - // The one most worth keeping, and the one a parse into today's struct cannot - // read: a phase this build has no name for, and a field it does not know. - ( - "a newer schema this build cannot parse at all", - serde_json::json!({ - "schema": 9_999, - "phase": "some_phase_from_the_future", - "first_start_unix": 1, - "something_this_build_never_heard_of": {"a": 1}, - }) - .to_string() - .into_bytes(), - ), - ] { - let dir = TempDir::new().expect("temp dir"); - std::fs::write(state_path(dir.path()), &bad).expect("plant the bad marker"); - - let first = MigrationState::load_or_create(dir.path(), MigrationPhase::Bridging); - let written = std::fs::read(state_path(dir.path())).expect("read it back"); - assert_ne!(written, bad, "{name}: the unusable marker was left on disk"); - if name.starts_with("a newer schema") { - // Moved aside rather than destroyed: it was written on purpose by a build - // that knew more than this one. - let mut kept = state_path(dir.path()).into_os_string(); - kept.push(".schema-9999"); - let kept = PathBuf::from(kept); - assert_eq!( - std::fs::read(&kept).expect("the newer marker must be kept"), - bad, - "the newer marker was written over rather than set aside" - ); - } - - // And the clock survives the next start, which is the point of all this. - let second = MigrationState::load_or_create(dir.path(), MigrationPhase::Bridging); - assert_eq!( - second.first_start_unix, first.first_start_unix, - "{name}: the shed hold restarted on the next boot" - ); - } - } - - /// A marker whose clock is impossible is corrected on disk too. - /// - /// `with_sane_clocks` fixes a future-dated marker for the process that read it. Left - /// there, the same correction is made again on every start, from a new now each time, - /// which is the same repeating reset by another route. - #[test] - fn a_future_dated_marker_is_corrected_on_disk_not_only_in_memory() { - let dir = TempDir::new().expect("temp dir"); - let ahead = now_unix().saturating_add(60 * 60 * 24 * 365); - let planted = serde_json::json!({ - "schema": 1, - "phase": "bridging", - "first_start_unix": ahead, - "committed_at_unix": null, - "rebuilds_since_commit": 0, - "shed_key_count": 0, - "kept_key_count": 0, - }) - .to_string(); - std::fs::write(state_path(dir.path()), &planted).expect("plant it"); - - let first = MigrationState::load_or_create(dir.path(), MigrationPhase::Bridging); - assert!( - first.first_start_unix < ahead, - "the clock should have been brought back to something possible" - ); - - let reread: MigrationState = - serde_json::from_slice(&std::fs::read(state_path(dir.path())).expect("read")) - .expect("the marker on disk must now parse"); - assert_eq!( - reread.first_start_unix, first.first_start_unix, - "the correction was made in memory and not written back" - ); - } - - /// A marker that is already right is not rewritten on every start. - /// - /// The counterpart to the two above: persisting on disagreement must not turn into - /// persisting unconditionally, which would put a write on every node's start path for - /// nothing. - #[test] - fn a_usable_marker_is_left_exactly_as_it_is() { - let dir = TempDir::new().expect("temp dir"); - let first = MigrationState::load_or_create(dir.path(), MigrationPhase::Bridging); - - // Written back out by hand in a shape this build would never produce: the same - // facts, different spacing and key order. Comparing the bytes of a marker this - // build wrote against the bytes after a second start proves nothing, because an - // unconditional rewrite produces the same bytes and the test passes either way. - // Something semantically equal but textually different is the only thing that can - // tell "left alone" from "written again". - let noncanonical = format!( - "{{\"kept_key_count\":{},\"shed_key_count\":{},\"rebuilds_since_commit\":{},\ - \"committed_at_unix\":null,\"first_start_unix\":{},\"phase\":\"bridging\",\ - \"schema\":{}}}", - first.kept_key_count, - first.shed_key_count, - first.rebuilds_since_commit, - first.first_start_unix, - first.schema - ); - std::fs::write(state_path(dir.path()), &noncanonical).expect("write"); - - MigrationState::load_or_create(dir.path(), MigrationPhase::Bridging); - assert_eq!( - std::fs::read_to_string(state_path(dir.path())).expect("read"), - noncanonical, - "a marker that already said the right thing was rewritten for no reason" - ); - } - - /// Two nodes on one disk take the same lock, and the override is what makes that true - /// where they do not share a `/tmp`. - /// - /// `lock_path_for` had no coverage at all, which is how it came to be right in a way - /// that is false on our own fleet: every shipped test sets `lock_dir` and so never - /// calls it. What the lock is for is one node copying at a time on a shared disk, so - /// what has to be true is that two different node roots on one volume produce one path. - /// - /// Nothing here touches process-wide environment. An earlier version of this test set - /// `TMPDIR` to stage the private-`/tmp` case, which moved every other test's temporary - /// directory and then deleted it underneath them: sixty-three unrelated tests failed - /// with LMDB errors that pointed nowhere near the cause. - #[test] - fn nodes_on_one_volume_agree_on_a_lock_path_when_they_are_told_where_it_is() { - let volume = TempDir::new().expect("temp dir"); - let a = volume.path().join("node-0"); - let b = volume.path().join("node-1"); - std::fs::create_dir_all(&a).expect("mkdir"); - std::fs::create_dir_all(&b).expect("mkdir"); - - // With no override and one shared temporary directory, which is the case the - // default is right for: two roots on one volume, one lock. - assert_eq!( - lock_path_with(&a, None), - lock_path_with(&b, None), - "two roots on one volume must share a lock when they share a /tmp" - ); - - // Where they do not share one, the default gives each node a lock of its own and - // every one of them takes it. That is the deployment hazard, and it is why the - // override exists rather than something the code can detect. - assert_ne!( - lock_path_with(&a, Some("/tmp/private-to-node-0")), - lock_path_with(&b, Some("/tmp/private-to-node-1")), - "different lock directories must give different locks, or the override would \ - not be able to express anything" - ); - - // And told where it lives, both land on it whatever their own root is. - let told = volume.path().to_string_lossy().into_owned(); - let shared_a = lock_path_with(&a, Some(&told)); - let shared_b = lock_path_with(&b, Some(&told)); - assert_eq!( - shared_a, shared_b, - "nodes told where the lock lives must all use it" - ); - assert!(shared_a.starts_with(volume.path()), "and use the one named"); - assert_eq!( - lock_path_with(&a, Some(" ")), - lock_path_with(&a, None), - "an empty setting is not a location and must not be treated as one" - ); - - // And the lock itself then does its job across the two roots. - let LockAttempt::Acquired(held) = VolumeLock::try_acquire(&a, Some(volume.path())) else { - panic!("the first node must take it"); - }; - assert!( - matches!( - VolumeLock::try_acquire(&b, Some(volume.path())), - LockAttempt::Busy - ), - "the second must wait rather than copy alongside it" - ); - drop(held); - assert!(matches!( - VolumeLock::try_acquire(&b, Some(volume.path())), - LockAttempt::Acquired(_) - )); - } - - /// A lock directory that cannot be written is reported, not silently ignored. - /// - /// The override is a deployment fact, so a typo in it must not read as "no lock needed - /// here". `Unavailable` is the honest answer and it already warns; what this pins is - /// that a bad override does not quietly fall back to a path that would appear to work. - #[test] - fn a_lock_directory_that_does_not_exist_is_unavailable_rather_than_ignored() { - let dir = TempDir::new().expect("temp dir"); - let missing = dir.path().join("no-such-directory"); - assert!(matches!( - VolumeLock::try_acquire(dir.path(), Some(&missing)), - LockAttempt::Unavailable - )); - } - - #[tokio::test] - async fn the_driver_exits_immediately_when_there_is_nothing_to_migrate() { - // The whole feature hangs off `run` being reachable from node startup. A port that - // dropped that call once already, and nothing caught it, because a fresh node has - // no legacy environment and every test built one directly. This asserts the entry - // point is callable and terminates on its own for a node with nothing to do. - let dir = TempDir::new().expect("temp dir"); - let store = Arc::new( - crate::storage::ChunkStore::new(crate::storage::ChunkStoreConfig { - root_dir: dir.path().to_path_buf(), - ..crate::storage::ChunkStoreConfig::test_default() - }) - .await - .expect("open store"), - ); - assert!(!store.has_legacy()); - - let context = MigrationContext { - p2p: None, - self_id: None, - self_xor: None, - commitment: None, - replication: None, - sync_state: None, - audit_challenge_coordinator: None, - peer_commitments: None, - close_group_size: 7, - }; - tokio::time::timeout( - Duration::from_secs(5), - run(store, context, CancellationToken::new()), - ) - .await - .expect("the driver must return rather than idle when there is nothing to migrate"); - } - - #[tokio::test] - async fn a_node_with_no_view_of_the_network_gives_up_nothing() { - // Every field is `None`, which is what a devnet or a node whose routing is not up - // yet looks like. No view of the network is no evidence, and the answer has to be - // "keep everything" rather than "nobody is closer, so give it all away". - let dir = TempDir::new().expect("temp dir"); - let store = Arc::new( - crate::storage::ChunkStore::new(crate::storage::ChunkStoreConfig { - root_dir: dir.path().to_path_buf(), - ..crate::storage::ChunkStoreConfig::test_default() - }) - .await - .expect("open store"), - ); - let context = MigrationContext { - p2p: None, - self_id: None, - self_xor: None, - commitment: None, - replication: None, - sync_state: None, - audit_challenge_coordinator: None, - peer_commitments: None, - close_group_size: 7, - }; - - let keys = vec![[1u8; 32], [2u8; 32], [3u8; 32]]; - let unconfirmed = unconfirmed_by_neighbours(&store, &context, &keys).await; - assert_eq!( - unconfirmed, keys, - "with no routing state every key must count as unproven" - ); - for key in &keys { - assert!( - !context.may_shed(key).await, - "and none of them may be given up" - ); - } - } - - #[test] - fn the_possession_threshold_comes_from_the_whole_group_not_the_qualifying_subset() { - use crate::replication::pruning::{prune_proofs_needed, target_peers_reported_present}; - use std::collections::{HashMap, HashSet}; - - // Seven holders. Deriving the bar from whichever peers happen to qualify is how - // two last holders destroy a chunk between them: each sees only the other - // publishing, so each needs exactly one proof, each gets it from the other, and - // both delete. The bar must come from the group. - let key = [7u8; 32]; - let group: Vec = (0..6u8).map(peer_id).collect(); - let only_one_qualifies: Vec = group.iter().take(1).copied().collect(); - - // That one peer does answer the challenge. - let mut proofs: HashMap> = HashMap::new(); - proofs.insert(key, only_one_qualifies.iter().copied().collect()); - - // The dangerous reading: bar taken from the qualifying subset, so one is enough. - assert!( - target_peers_reported_present( - &key, - &only_one_qualifies, - &proofs, - prune_proofs_needed(only_one_qualifies.len()), - ), - "this is the mistake being guarded against, shown here to be a real risk" - ); - - // The correct reading: bar taken from the whole group, so one is nowhere near. - assert!( - !target_peers_reported_present( - &key, - &only_one_qualifies, - &proofs, - prune_proofs_needed(group.len()), - ), - "one proof must never satisfy a group of six" - ); - - // And with the whole group answering, it passes. - proofs.insert(key, group.iter().copied().collect()); - assert!(target_peers_reported_present( - &key, - &group, - &proofs, - prune_proofs_needed(group.len()), - )); - } - - #[test] - #[serial] - fn shedding_reads_the_same_switch_the_auditors_read() { - use crate::replication::config::{ - close_group_storage_penalty_suspended, set_close_group_storage_penalty_suspended, - }; - // One switch, not two. There used to be a second constant of the same name with - // its own environment override on this side of the decision, and nothing coupling - // them: a node could have been willing to shed while every peer still applied the - // full penalty, which is precisely what the release ordering exists to prevent. - set_close_group_storage_penalty_suspended(true); - assert!(close_group_storage_penalty_suspended()); - set_close_group_storage_penalty_suspended(false); - assert!(!close_group_storage_penalty_suspended()); - set_close_group_storage_penalty_suspended( - crate::replication::config::RELEASE_SUSPEND_CLOSE_GROUP_STORAGE_PENALTY, - ); - } - - #[test] - fn a_close_group_is_split_into_enough_waves_to_bound_concurrent_migrations() { - // Seven holders, two at a time, is four waves. - assert_eq!(migration_wave_count(7), 4); - assert_eq!(migration_wave_count(2), 1); - assert_eq!(migration_wave_count(1), 1); - // A degenerate width must still yield a usable wave count rather than dividing by - // zero or collapsing to "everyone at once". - assert_eq!(migration_wave_count(0), 1); - } - - #[test] - fn wave_assignment_is_stable_per_node_and_spread_across_the_group() { - use std::collections::HashMap; - let waves = migration_wave_count(7); - - // The same node always gets the same wave: a restart must not move a node into a - // turn that has already passed. - let peer = peer_id(7); - assert_eq!( - migration_wave_for(Some(&peer), 7), - migration_wave_for(Some(&peer), 7) - ); - - // And across many nodes every wave is used, so the group is genuinely staggered - // rather than all landing together. - let mut counts: HashMap = HashMap::new(); - for b in 0..=255u8 { - let w = migration_wave_for(Some(&peer_id(b)), 7); - assert!(w < waves, "wave {w} outside 0..{waves}"); - *counts.entry(w).or_default() += 1; - } - assert_eq!( - counts.len() as u64, - waves, - "every wave should be occupied, got {counts:?}" - ); - } - - #[test] - fn waves_are_actually_staggered_under_the_shipped_defaults() { - // The combination is what matters, not either setting alone. Measured from first - // start, a 72 hour hold and 24 hour waves cancel out: waves would open at 0, 24, - // 48 and 72 hours while nothing may shed until 72, so every wave is open the - // moment the first one can act and the whole close group moves together. Measured - // from the end of the hold, they stagger as intended. - let config = MigrationConfig::default(); - assert_eq!(config.shed_hold_hours, 72); - assert_eq!(config.wave_hours, 24); - - let mut state = MigrationState::new(MigrationPhase::Bridging); - let waves = migration_wave_count(7); - assert_eq!(waves, 4); - - // Nothing is open before the hold ends. - state.first_start_unix = now_unix(); - for w in 0..waves { - assert!( - !wave_has_opened(&state, &config, w), - "wave {w} opened too early" - ); - } - - // At the end of the hold, exactly the first wave is open. - state.first_start_unix = now_unix().saturating_sub(72 * 3600 + 60); - assert!(wave_has_opened(&state, &config, 0)); - for w in 1..waves { - assert!( - !wave_has_opened(&state, &config, w), - "wave {w} must wait its turn, or the group migrates together" - ); - } - - // Each later wave opens one wave_hours after the one before it. - for open in 1..waves { - state.first_start_unix = now_unix().saturating_sub((72 + open * 24) * 3600 + 60); - for w in 0..=open { - assert!(wave_has_opened(&state, &config, w)); - } - for w in open + 1..waves { - assert!(!wave_has_opened(&state, &config, w)); - } - } - } - - #[test] - fn an_implausible_clock_restarts_the_holds_rather_than_voiding_them() { - let dir = TempDir::new().expect("temp dir"); - let config = MigrationConfig::default(); - - // Zero is what a node with an unsynchronised clock writes at first boot, and it - // would make every hold vacuous. So would a time in the future. - let mut state = MigrationState::new(MigrationPhase::Committed); - state.first_start_unix = 0; - state.committed_at_unix = Some(0); - state.save(dir.path()).expect("save"); - - let loaded = MigrationState::load_or_new(dir.path(), MigrationPhase::Bridging); - assert!( - !loaded.shed_hold_elapsed(&config), - "the hold must not be void" - ); - assert!(!loaded.retire_delay_elapsed(&config)); - - let mut future = MigrationState::new(MigrationPhase::Committed); - future.first_start_unix = now_unix().saturating_add(10 * 365 * 24 * 3600); - future.committed_at_unix = Some(future.first_start_unix); - future.save(dir.path()).expect("save"); - let loaded = MigrationState::load_or_new(dir.path(), MigrationPhase::Bridging); - assert!(!loaded.shed_hold_elapsed(&config)); - assert!(!loaded.retire_delay_elapsed(&config)); - } - - #[test] - fn copy_reports_accumulate_across_passes() { - let mut total = CopyReport::default(); - total.merge(CopyReport { - copied: 3, - bytes: 300, - ..CopyReport::default() - }); - total.merge(CopyReport { - copied: 2, - bytes: 200, - unusable: 1, - stopped_for_space: true, - ..CopyReport::default() - }); - assert_eq!(total.copied, 5); - assert_eq!(total.bytes, 500); - assert_eq!(total.unusable, 1); - assert!(total.stopped_for_space); - } - /// A peer that received the commitment and then left the group is not evidence. - /// - /// It is not going to audit this node, so counting it lets a node give chunks up - /// while the neighbours who will audit it still hold it to the old, larger key set. - #[test] - fn a_departed_peer_that_knows_the_commitment_does_not_open_the_gate() { - let received: HashSet = (0..6).map(peer_id).collect(); - let still_here: Vec = (0..3).map(peer_id).collect(); - let joined_since: Vec = (100..103).map(peer_id).collect(); - let current: Vec = still_here - .iter() - .chain(joined_since.iter()) - .copied() - .collect(); - - // Six peers know it and the group is six wide, so a count that ignores who is - // actually here would sail past the threshold. - assert_eq!(received.len(), 6); - assert_eq!(current.len(), 6); - assert!(!enough_of_the_group_knows(&received, ¤t, 5)); - - // Only the three that are both here and informed count. - assert!(enough_of_the_group_knows(&received, ¤t, 3)); - assert!(!enough_of_the_group_knows(&received, ¤t, 4)); - } - - #[test] - fn a_group_that_has_all_seen_the_commitment_opens_the_gate() { - let group: Vec = (0..6).map(peer_id).collect(); - let received: HashSet = group.iter().copied().collect(); - assert!(enough_of_the_group_knows(&received, &group, 5)); - } - - #[test] - fn no_peer_ever_satisfies_a_zero_threshold() { - let group: Vec = (0..6).map(peer_id).collect(); - let received: HashSet = group.iter().copied().collect(); - // A group this node cannot reason about must not be read as unanimous consent. - assert!(!enough_of_the_group_knows(&received, &group, 0)); - } - - /// The gate stays shut when routing cannot show a full close group at all. - /// - /// Without a routing view there is no way to tell an informed neighbour from a - /// departed one, and an unanswerable question must not read as a yes. - #[tokio::test] - async fn without_a_routing_view_the_commitment_gate_stays_shut() { - let context = MigrationContext { - p2p: None, - self_id: None, - self_xor: None, - commitment: None, - replication: None, - sync_state: None, - audit_challenge_coordinator: None, - peer_commitments: None, - close_group_size: 7, - }; - assert!(!context.neighbours_know_the_commitment().await); - } -} diff --git a/src/storage/migration_signal.rs b/src/storage/migration_signal.rs index e698561b..cd219d31 100644 --- a/src/storage/migration_signal.rs +++ b/src/storage/migration_signal.rs @@ -29,7 +29,7 @@ //! about the filesystem, so it is asked of the filesystem, whatever the node was configured //! to do with it. -use std::path::Path; +use std::path::{Path, PathBuf}; use std::sync::{Arc, Weak}; use std::time::Duration; @@ -51,13 +51,13 @@ use crate::logging::{info, warn}; const REPORT_INTERVAL: Duration = Duration::from_mins(15); /// The directory the old chunk store lives in. -const LEGACY_ENV_DIR: &str = "chunks.mdb"; +pub const LEGACY_ENV_DIR: &str = "chunks.mdb"; /// What retirement renames it to before deleting it. -const RETIRED_SUFFIX: &str = ".retired"; +pub(super) const RETIRED_SUFFIX: &str = ".retired"; /// The file retirement writes inside a directory to say it has finished with it. -const RETIRED_MARKER: &str = "RETIRED"; +pub(super) const RETIRED_MARKER: &str = "RETIRED"; /// The token that carries the state, so a reader can find it wherever it sits. const SIGNAL_PREFIX: &str = "migration/"; @@ -109,8 +109,12 @@ impl MigrationSignal { } /// What one leftover directory means for the node carrying it. +/// +/// `legacy_artifacts` decides what it may delete from this same verdict, so that the state a +/// node reports and the state its cleanup acts on can never be two different readings of one +/// directory. #[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum Leftover { +pub(super) enum Leftover { /// It holds chunks this node has not moved. Holding, /// It holds nothing, or it carries the mark that says it was finished with. @@ -128,7 +132,7 @@ enum Leftover { /// /// An entry that cannot be read is returned rather than skipped, so it becomes `Unknown` /// rather than silently becoming `Files`. -fn legacy_directories(root_dir: &Path) -> Result, Unreadable> { +pub(super) fn legacy_directories(root_dir: &Path) -> Result, Unreadable> { let mut found = Vec::new(); // `symlink_metadata`, not `try_exists`: the latter follows links, so a dangling or @@ -169,10 +173,10 @@ fn legacy_directories(root_dir: &Path) -> Result, Unread /// There is no answer to give about this root. #[derive(Clone, Copy, Debug, PartialEq, Eq)] -struct Unreadable; +pub(super) struct Unreadable; /// The most tombstones one root can hold, matching what retirement will ever create. -const MAX_TOMBSTONES: u32 = 64; +pub(super) const MAX_TOMBSTONES: u32 = 64; /// Is this a name retirement gives a tombstone? /// @@ -180,7 +184,7 @@ const MAX_TOMBSTONES: u32 = 64; /// writes it. The bounds are not decoration: retirement only ever counts up to 64, so `.65` /// and `.007` are names it cannot have produced, and this list becomes a list of directories /// a later release deletes. -fn is_tombstone_name(name: &str) -> bool { +pub(super) fn is_tombstone_name(name: &str) -> bool { let base = format!("{LEGACY_ENV_DIR}{RETIRED_SUFFIX}"); if name == base { return true; @@ -197,7 +201,7 @@ fn is_tombstone_name(name: &str) -> bool { } /// What one directory says about itself. -fn classify(dir: &Path) -> Leftover { +pub(super) fn classify(dir: &Path) -> Leftover { match std::fs::symlink_metadata(dir) { // A link is never treated as finished with, whatever it points at: the mark would // have been written through it into a directory that is not this node's. It is also @@ -217,7 +221,7 @@ fn classify(dir: &Path) -> Leftover { Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} Err(_) => return Leftover::Unreadable, } - // No mark. A directory with nothing in it holds no chunks, so it cannot be hiding any: + // No mark. A directory with nothing in it is hiding nothing, whatever it once held: // that is what a cleanup interrupted between emptying a tombstone and removing it // leaves behind. std::fs::read_dir(dir).map_or(Leftover::Unreadable, |mut entries| { diff --git a/src/storage/mod.rs b/src/storage/mod.rs index fc493393..8d037758 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -1,8 +1,8 @@ //! Storage subsystem for chunk persistence. //! -//! This module provides content-addressed LMDB storage for chunks, -//! along with a protocol handler that integrates with saorsa-core's -//! `Protocol` trait for automatic message routing. +//! This module provides content-addressed storage for chunks, one immutable file per +//! chunk, along with a protocol handler that integrates with saorsa-core's `Protocol` +//! trait for automatic message routing. //! //! # Architecture //! @@ -19,7 +19,7 @@ //! │ QuoteRequest ChunkPutRequest ChunkGetRequest //! │ │ │ │ │ //! │ ▼ ▼ ▼ │ -//! │ QuoteGenerator PaymentVerifier LmdbStorage│ +//! │ QuoteGenerator PaymentVerifier ChunkStore│ //! │ │ │ │ │ //! │ └─────────────────────────┴─────────────────┘ │ //! │ │ │ @@ -31,11 +31,11 @@ //! //! ```rust,ignore //! use std::sync::Arc; -//! use ant_node::storage::{AntProtocol, LmdbStorage, LmdbStorageConfig}; +//! use ant_node::storage::{AntProtocol, ChunkStore, ChunkStoreConfig}; //! //! // Create storage -//! let config = LmdbStorageConfig::default(); -//! let storage = Arc::new(LmdbStorage::new(config).await?); +//! let config = ChunkStoreConfig::default(); +//! let storage = Arc::new(ChunkStore::new(config).await?); //! //! // Create protocol handler //! let protocol = AntProtocol::new(storage, Arc::new(payment_verifier), Arc::new(quote_generator)); @@ -44,27 +44,36 @@ //! listener.register_protocol(protocol).await?; //! ``` -pub(crate) mod chunk_store; +// `test-utils` makes this module public so integration tests and downstream harnesses can +// reach the store directly, and that is not a narrow door: with the feature on, every `pub` +// item inside it — `StoreLayout`, `CapacityVerdict`, every inherent method on `ChunkStore` — +// is importable from outside the crate. The re-export list below is the boundary for the +// DEFAULT build only. Anything relying on the wider surface is relying on a test feature, and +// this comment is here so that is a decision rather than a discovery. #[cfg(any(test, feature = "test-utils"))] -pub mod file_store; +pub mod chunk_store; #[cfg(not(any(test, feature = "test-utils")))] -pub(crate) mod file_store; +pub(crate) mod chunk_store; mod handler; -pub(crate) mod lmdb; -pub mod migration; +// Both are this crate's own business. The cleanup is called once, from the node builder, and +// the signal was `pub(crate)` in the release that added it; exporting either would publish a +// migration this release exists to finish. +pub(crate) mod legacy_artifacts; +// Carried forward from the release before this one. Without it a node on this release reads +// to its peers as one that never reported at all, and the fleet gate that authorised this +// release could never come back clean again. pub(crate) mod migration_signal; pub(crate) mod traffic; pub use crate::ant_protocol::XorName; -pub use chunk_store::{ChunkStore, ChunkStoreConfig, VerifyReport, LEGACY_ENV_DIR}; -pub(crate) use file_store::{rename_with_retry, Reservation}; -pub use file_store::{FileStore, FileStoreConfig, StoreLayout}; +pub(crate) use chunk_store::{rename_with_retry, Reservation}; +pub use chunk_store::{ChunkStore, ChunkStoreConfig}; +// Crate-private, as it was before the two stores became one: `CapacityVerdict` was +// `pub(crate)` on the old store and has no caller outside this crate. +pub(crate) use chunk_store::CapacityVerdict; pub use handler::AntProtocol; pub(crate) use handler::ChunkRequestContext; pub use handler::SELF_CLOSENESS_GATE_WIDTH; -pub(crate) use lmdb::CapacityVerdict; -pub use lmdb::{LmdbStorage, LmdbStorageConfig}; -pub use migration::{MigrationConfig, MigrationPhase, MigrationState}; /// Bytes in one MiB. pub const MIB: u64 = 1024 * 1024; diff --git a/tests/chunk_store_crash_safety.rs b/tests/chunk_store_crash_safety.rs new file mode 100644 index 00000000..b153942a --- /dev/null +++ b/tests/chunk_store_crash_safety.rs @@ -0,0 +1,250 @@ +//! What survives a process dying part-way through a write. +//! +//! The store rests on being able to stop at any moment and start again: every step is +//! idempotent and re-derived from the filesystem. That is easy to assert and hard to +//! believe without trying it, so these tests kill a real child process at a real point in +//! the work and then open the store in this one and check what is there. +//! +//! **What this does and does not prove.** A killed process loses nothing the kernel has +//! already accepted, so this covers ordering and bookkeeping: a chunk is whole or absent +//! and never half-indexed, and what an interrupted write leaves behind is swept. It does +//! not cover power loss, where the kernel loses what it accepted and never wrote. That is +//! still a fleet gate. +//! +//! These tests came from the harness that proved the migration off the old chunk store. +//! Most of that harness went with the migration; these two did not belong to it. They are +//! about the store's own publish path, which is now the only one there is. + +#![allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + clippy::missing_panics_doc, + clippy::cast_possible_truncation +)] + +use ant_node::storage::{ChunkStore, ChunkStoreConfig}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::time::Duration; +use tempfile::TempDir; + +/// Run this test binary again as a child in the mode named by `role`, wait until it has +/// reached the named point, and kill it there. +/// +/// A child process rather than a thread, because the point is to lose everything the +/// process was holding: buffers, in-memory index, locks, half-finished intentions. +/// +/// The wait is a handshake, not a sleep. An earlier version of this slept and hoped, and +/// on a quick machine the child had finished everything before the kill arrived, so the +/// test was checking a clean shutdown while claiming to check a crash. The child now stops +/// at a failpoint inside the write and says so by writing a marker; this waits for the +/// marker and then kills it, so the process always dies at the same point in the same +/// operation. +fn kill_child_at_failpoint(role: &str, root: &Path, failpoint: &str, let_through: u64) -> PathBuf { + let marker = root.join(format!("reached-{role}")); + let _ = std::fs::remove_file(&marker); + + let exe = std::env::current_exe().expect("this test binary"); + let mut child = Command::new(exe) + .arg("--exact") + .arg(role) + .arg("--nocapture") + .arg("--ignored") + .env("ANT_CRASH_TEST_ROOT", root) + .env(failpoint, &marker) + .env( + ant_node::storage::chunk_store::HALT_AFTER, + let_through.to_string(), + ) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()) + .spawn() + .expect("spawn the child"); + + // Generous, but not unbounded. Without a deadline a failpoint that stopped working + // would hang the job rather than fail it, and a hang says nothing about the code. + let deadline = std::time::Instant::now() + Duration::from_mins(2); + while !marker.exists() { + if let Ok(Some(status)) = child.try_wait() { + panic!("the child exited before reaching the failpoint: {status}"); + } + if std::time::Instant::now() > deadline { + let _ = child.kill(); + panic!("the child never reached the failpoint"); + } + std::thread::sleep(Duration::from_millis(10)); + } + + child.kill().expect("kill the child"); + let _ = child.wait(); + marker +} + +/// Where the child was told to work. +fn child_root() -> PathBuf { + PathBuf::from(std::env::var("ANT_CRASH_TEST_ROOT").expect("the child needs a root")) +} + +/// Child mode: write chunks into a file store until killed. +#[tokio::test] +#[ignore = "child process of a crash test, not run on its own"] +async fn child_writes_until_killed() { + let root = child_root(); + let store = ChunkStore::new(ChunkStoreConfig { + root_dir: root, + disk_reserve: 0, + ..ChunkStoreConfig::default() + }) + .await + .expect("open"); + + // Always a chunk it has not written before, so the kill lands in real work rather + // than in a re-offer of something already on disk. An earlier version cycled the same + // hundred keys and spent almost all its time confirming duplicates. + let mut n = 0usize; + loop { + let content = chunk_bytes(n); + let address = ant_node::client::compute_address(&content); + let _ = store.put(&address, &content).await; + n += 1; + } +} + +/// A process killed inside a publish leaves no chunk it cannot serve. +/// +/// The child is stopped at the last moment before the chunk's name exists on disk: on Unix +/// the bytes written to a temporary file with the rename not yet made, off Unix the point +/// before the file is created at all, since that platform writes under the final name +/// because a rename there carries no durability guarantee. The failure this guards against +/// is the same on both: a name outliving its bytes. The index is built from filenames at +/// startup, so a partial file wearing a real chunk name would be advertised, committed to, +/// and unservable. +#[tokio::test] +async fn a_process_killed_mid_publish_leaves_no_chunk_it_cannot_serve() { + let tmp = TempDir::new().expect("temp dir"); + let root = tmp.path().join("node"); + std::fs::create_dir_all(&root).expect("mkdir"); + + // Twenty chunks land before the crash, so the store this reopens has real content in + // it. Stopping the very first write would leave nothing indexed and the loop below + // would pass by iterating over nothing. + let marker = kill_child_at_failpoint( + "child_writes_until_killed", + &root, + ant_node::storage::chunk_store::HALT_BEFORE_PUBLISH, + 20, + ); + assert!( + marker.exists(), + "the child must have reached the failpoint before it was killed" + ); + + // Reopening is itself part of the assertion: a store that cannot start after a crash + // is a node that cannot start. + let store = reopen(&root).await; + // The child discards its put results and the failpoint counts arrivals, not successes, + // so every publish before the kill could in principle have failed. An empty store makes + // the loop below pass over nothing, which is the one outcome that would let this test + // report success having checked no chunk at all. + let held = store.all_keys().await.expect("all_keys"); + assert!( + !held.is_empty(), + "the child published nothing before it was killed, so there is nothing to check" + ); + for key in held { + let served = store.get(&key).await; + assert!( + matches!(served, Ok(Some(_))), + "chunk {} is claimed after a crash but cannot be served: {served:?}", + hex::encode(key) + ); + } +} + +/// The temporary file a killed publish left behind is swept, not indexed. +/// +/// It carries no chunk name, so it can never be served, and leaving it would cost disk +/// for the life of the node. +/// +/// Unix only, because the leftover only exists on Unix. Off Unix the store creates the +/// file under its final name and flushes it, deliberately, since a rename there is not +/// documented to be durable. So there is no temporary file to sweep and the equivalent +/// hazard is different: a real chunk name over bytes that are short or wrong. That one is +/// covered by the store's own tests, which run on every platform: a read verifies the bytes +/// against the name and refuses a file that does not match. There used to be a second answer +/// here, the re-hash-everything pass retirement ran before deleting, and this release deletes +/// retirement, so it is no longer one. Forced power loss on a real filesystem remains an open +/// gate, named in ADR-0022. +#[cfg(unix)] +#[tokio::test] +async fn the_leftovers_of_a_killed_publish_are_swept() { + let tmp = TempDir::new().expect("temp dir"); + let root = tmp.path().join("node"); + std::fs::create_dir_all(&root).expect("mkdir"); + + kill_child_at_failpoint( + "child_writes_until_killed", + &root, + ant_node::storage::chunk_store::HALT_BEFORE_PUBLISH, + 5, + ); + + let before = temp_files(&root.join("chunks")); + assert!( + before > 0, + "the child should have left a temporary file behind when it was killed" + ); + + let store = reopen(&root).await; + store.wait_idle().await; + assert_eq!( + temp_files(&root.join("chunks")), + 0, + "the store should sweep what an interrupted write left" + ); + drop(store); +} + +/// How many partly-written files are under `chunks_dir`. +#[cfg(unix)] +fn temp_files(chunks_dir: &Path) -> usize { + let Ok(shards) = std::fs::read_dir(chunks_dir) else { + return 0; + }; + shards + .flatten() + .filter_map(|shard| std::fs::read_dir(shard.path()).ok()) + .flat_map(std::iter::IntoIterator::into_iter) + .flatten() + .filter(|entry| { + entry + .file_name() + .to_str() + .is_some_and(|name| !name.chars().all(|c| c.is_ascii_hexdigit())) + }) + .count() +} + +/// Open the store the way a restart would. +async fn reopen(root: &Path) -> ChunkStore { + let config = ChunkStoreConfig { + root_dir: root.to_path_buf(), + disk_reserve: 0, + ..ChunkStoreConfig::default() + }; + ChunkStore::new(config) + .await + .expect("the store must open after a crash") +} + +/// Deterministic content for chunk `n`. `n` goes in verbatim so no two differ only by a +/// wrap and collapse into one chunk. +fn chunk_bytes(n: usize) -> Vec { + let mut content = vec![0u8; 4096]; + content[..8].copy_from_slice(&(n as u64).to_le_bytes()); + for (i, byte) in content.iter_mut().enumerate().skip(8) { + *byte = ((i.wrapping_mul(17)).wrapping_add(n) % 251) as u8; + } + content +} diff --git a/tests/e2e/data_types/chunk.rs b/tests/e2e/data_types/chunk.rs index 2c875b76..650b8e08 100644 --- a/tests/e2e/data_types/chunk.rs +++ b/tests/e2e/data_types/chunk.rs @@ -358,7 +358,7 @@ mod tests { // so all Arc clones are released. // 2. Abort the protocol task that holds an Arc. // 3. Drop the node's own Arc. - // This ensures the LMDB env is fully closed before reopening. + // This ensures the chunk store is fully closed before reopening. let data_dir = { let node = harness .network_mut() diff --git a/tests/e2e/fresh_offer_capacity.rs b/tests/e2e/fresh_offer_capacity.rs index 5774c2a0..5b3fcecd 100644 --- a/tests/e2e/fresh_offer_capacity.rs +++ b/tests/e2e/fresh_offer_capacity.rs @@ -47,7 +47,7 @@ const UPLOAD_CHUNKS: usize = 48; /// one test process, so full-size chunks across every node would dominate the /// harness's memory. An admission slot is held per *offer* regardless of /// payload size, so the count above is what stresses the ceiling. The -/// trade-off is that the receiver's LMDB write is faster than production's, +/// trade-off is that the receiver's write is faster than production's, /// which is part of why this is a lower bound. const CHUNK_BYTES: usize = 64 * 1024; diff --git a/tests/e2e/replication.rs b/tests/e2e/replication.rs index abf5f92b..adc13dda 100644 --- a/tests/e2e/replication.rs +++ b/tests/e2e/replication.rs @@ -733,7 +733,7 @@ async fn test_paid_list_persistence() { dir }; - // Shut down the replication engine so the LMDB env is released + // Shut down the replication engine so the chunk store is released { let node = harness.network_mut().node_mut(3).expect("node"); if let Some(ref mut engine) = node.replication_engine { @@ -2916,7 +2916,7 @@ async fn scenario_43_paid_list_persists_across_restart() { dir }; - // Shut down the replication engine so the LMDB env is released + // Shut down the replication engine so the chunk store is released { let node = harness.network_mut().node_mut(3).expect("node"); if let Some(ref mut engine) = node.replication_engine { diff --git a/tests/e2e/subtree_audit_testnet.rs b/tests/e2e/subtree_audit_testnet.rs index ff8f86f2..efb05a0c 100644 --- a/tests/e2e/subtree_audit_testnet.rs +++ b/tests/e2e/subtree_audit_testnet.rs @@ -3,7 +3,7 @@ //! //! These spin a real multi-node testnet and drive the SHIPPED audit over the //! live wire (real `handle_subtree_challenge` responder + `run_subtree_audit` -//! auditor + real LMDB storage), via the test-only `audit_peer_now` / +//! auditor + a real chunk store), via the test-only `audit_peer_now` / //! `rebuild_commitment_now` engine hooks. They prove the two outcomes that //! matter for a testnet: //! diff --git a/tests/e2e/testnet.rs b/tests/e2e/testnet.rs index 6e878709..f0061f66 100644 --- a/tests/e2e/testnet.rs +++ b/tests/e2e/testnet.rs @@ -11,7 +11,7 @@ //! - Message encoding/decoding (postcard serialization) //! - Content address verification //! - Payment verification (when enabled) -//! - LMDB storage persistence +//! - chunk store persistence use ant_node::ant_protocol::{ ChunkGetRequest, ChunkGetResponse, ChunkMessage, ChunkMessageBody, ChunkPutRequest, @@ -1051,7 +1051,7 @@ impl TestNetwork { /// Create a test node (but don't start it yet). /// /// Initializes the `AntProtocol` handler with: - /// - LMDB storage in the node's data directory + /// - the chunk store in the node's data directory /// - Payment verification configured per `TestNetworkConfig` /// - Quote generation with a test rewards address async fn create_node( @@ -1110,18 +1110,18 @@ impl TestNetwork { /// Create an `AntProtocol` handler for a test node. /// /// Configures: - /// - LMDB storage with verification enabled + /// - the chunk store with verification enabled /// - Payment verification (enabled/disabled based on `payment_enforcement`) /// - Quote generator with a test rewards address /// /// # Arguments /// - /// * `data_dir` - Directory for LMDB storage + /// * `data_dir` - Directory for the chunk store /// * `payment_enforcement` - Whether to enable EVM payment verification /// /// # Errors /// - /// Returns an error if LMDB storage initialisation fails. + /// Returns an error if the chunk store cannot be opened. pub async fn create_ant_protocol( data_dir: &std::path::Path, evm_network: Option, @@ -1134,14 +1134,14 @@ impl TestNetwork { /// /// # Errors /// - /// Returns an error if LMDB storage initialisation fails. + /// Returns an error if the chunk store cannot be opened. pub async fn create_ant_protocol_with_disk_reserve( data_dir: &std::path::Path, evm_network: Option, disk_reserve: u64, identity: &saorsa_core::identity::NodeIdentity, ) -> Result { - // Create LMDB storage + // Create the chunk store let storage_config = ChunkStoreConfig { root_dir: data_dir.to_path_buf(), disk_reserve, @@ -1149,7 +1149,7 @@ impl TestNetwork { }; let storage = ChunkStore::new(storage_config) .await - .map_err(|e| TestnetError::Core(format!("Failed to create LMDB storage: {e}")))?; + .map_err(|e| TestnetError::Core(format!("Failed to create the chunk store: {e}")))?; // Create payment verifier (EVM is always on). // When an EVM network is provided (e.g. Anvil), use it for on-chain verification. diff --git a/tests/migration_crash_safety.rs b/tests/migration_crash_safety.rs deleted file mode 100644 index 38d0ea37..00000000 --- a/tests/migration_crash_safety.rs +++ /dev/null @@ -1,542 +0,0 @@ -//! What survives a process dying part-way through the migration. -//! -//! The design rests on being able to stop at any moment and start again: every step is -//! idempotent and re-derived from the filesystem. That is easy to assert and hard to -//! believe without trying it, so these tests kill a real child process at a real point in -//! the work and then open the store in this one and check what is there. -//! -//! **What this does and does not prove.** A killed process loses nothing the kernel has -//! already accepted, so this covers ordering and bookkeeping: a chunk is whole or absent -//! and never half-indexed, what an interrupted write leaves behind is swept, and the store -//! always opens. It does not cover losing the page cache, which is what a real power cut -//! adds and what no hosted runner can do. That remains a fleet gate, and this is the part -//! of it that can be automated. -//! -//! The children stop at a named failpoint and say so, and the parent kills them there. An -//! earlier version slept and hoped; on a quick machine the child had finished before the -//! kill arrived, so the test was checking a clean shutdown while claiming to check a -//! crash. -//! -//! Runs on every platform CI covers, which is the filesystem matrix that matters: ext4, -//! APFS and NTFS. - -#![allow( - clippy::unwrap_used, - clippy::expect_used, - clippy::panic, - clippy::missing_panics_doc, - // Test fixtures: every cast here is of a bounded loop counter into a byte, and the - // wrap is what makes the fill vary. - clippy::cast_possible_truncation -)] - -use ant_node::storage::{ChunkStore, ChunkStoreConfig, LmdbStorage, LmdbStorageConfig}; -use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; -use std::time::Duration; -use tempfile::TempDir; - -/// Chunks the child writes before it is killed. -const CHUNKS: usize = 120; - -/// Deterministic content for chunk `n`. `n` goes in verbatim so no two differ only by a -/// wrap and collapse into one chunk. -fn chunk_bytes(n: usize) -> Vec { - let mut content = vec![0u8; 4096]; - content[..8].copy_from_slice(&(n as u64).to_le_bytes()); - for (i, byte) in content.iter_mut().enumerate().skip(8) { - *byte = ((i.wrapping_mul(17)).wrapping_add(n) % 251) as u8; - } - content -} - -/// Run this test binary again as a child in the mode named by `role`, wait until it has -/// reached the named point, and kill it there. -/// -/// A child process rather than a thread, because the point is to lose everything the -/// process was holding: buffers, in-memory index, locks, half-finished intentions. -/// -/// The wait is a handshake, not a sleep. An earlier version of this slept and hoped, and -/// on a quick machine the child had finished everything before the kill arrived, so the -/// test was checking a clean shutdown while claiming to check a crash. The child now stops -/// at a failpoint inside the write and says so by writing a marker; this waits for the -/// marker and then kills it, so the process always dies at the same point in the same -/// operation. -fn kill_child_at_failpoint(role: &str, root: &Path, failpoint: &str, let_through: u64) -> PathBuf { - let marker = root.join(format!("reached-{role}")); - let _ = std::fs::remove_file(&marker); - - let exe = std::env::current_exe().expect("this test binary"); - let mut child = Command::new(exe) - .arg("--exact") - .arg(role) - .arg("--nocapture") - .arg("--ignored") - .env("ANT_CRASH_TEST_ROOT", root) - .env(failpoint, &marker) - .env( - ant_node::storage::file_store::HALT_AFTER, - let_through.to_string(), - ) - .stdout(Stdio::null()) - .stderr(Stdio::inherit()) - .spawn() - .expect("spawn the child"); - - // Generous, but not unbounded. Without a deadline a failpoint that stopped working - // would hang the job rather than fail it, and a hang says nothing about the code. - let deadline = std::time::Instant::now() + Duration::from_mins(2); - while !marker.exists() { - if let Ok(Some(status)) = child.try_wait() { - panic!("the child exited before reaching the failpoint: {status}"); - } - if std::time::Instant::now() > deadline { - let _ = child.kill(); - panic!("the child never reached the failpoint"); - } - std::thread::sleep(Duration::from_millis(10)); - } - - child.kill().expect("kill the child"); - let _ = child.wait(); - marker -} - -/// Where the child was told to work. -fn child_root() -> PathBuf { - PathBuf::from(std::env::var("ANT_CRASH_TEST_ROOT").expect("the child needs a root")) -} - -/// Child mode: write chunks into a file store until killed. -#[tokio::test] -#[ignore = "child process of a crash test, not run on its own"] -async fn child_writes_until_killed() { - let root = child_root(); - let store = ChunkStore::new(ChunkStoreConfig { - root_dir: root, - disk_reserve: 0, - ..ChunkStoreConfig::default() - }) - .await - .expect("open"); - - // Always a chunk it has not written before, so the kill lands in real work rather - // than in a re-offer of something already on disk. An earlier version cycled the same - // hundred keys and spent almost all its time confirming duplicates. - let mut n = 0usize; - loop { - let content = chunk_bytes(n); - let address = ant_node::client::compute_address(&content); - let _ = store.put(&address, &content).await; - n += 1; - } -} - -/// Child mode: copy a legacy environment into files until killed. -#[tokio::test] -#[ignore = "child process of a crash test, not run on its own"] -async fn child_migrates_until_killed() { - let root = child_root(); - let mut config = ChunkStoreConfig { - root_dir: root.clone(), - disk_reserve: 0, - ..ChunkStoreConfig::default() - }; - config.migration.tick_secs = 1; - config.migration.copier_throttle_mib_per_sec = 0; - config.migration.copier_slack_mb = 0; - config.migration.lock_dir = Some(root); - let store = ChunkStore::new(config).await.expect("open"); - - let shutdown = tokio_util::sync::CancellationToken::new(); - // One chunk at a time, so the kill lands between two of them rather than after the - // whole thing. Deliberately no sleep at the end: a child that finished and then idled - // would let this test pass having crashed nothing. - loop { - let keys = store.legacy_only_keys(); - let Some(key) = keys.first() else { - break; - }; - let _ = store.copy_batch(&[*key], 0, 0, &shutdown).await; - } - panic!("the child copied everything before it was killed, so nothing was interrupted"); -} - -/// Plant a legacy environment holding chunks numbered from `first`, and close it. -async fn seed_legacy_from(root: &Path, first: usize) -> Vec<(usize, [u8; 32])> { - let lmdb = LmdbStorage::new(LmdbStorageConfig { - root_dir: root.to_path_buf(), - verify_on_read: true, - max_map_size: 0, - disk_reserve: 0, - }) - .await - .expect("open legacy"); - let mut keys = Vec::new(); - for n in first..first + CHUNKS { - let content = chunk_bytes(n); - let address = ant_node::client::compute_address(&content); - lmdb.put(&address, &content).await.expect("seed"); - // Paired with its chunk number, because half of these are about to be deleted and - // a bare position in the surviving list no longer says which chunk it is. - keys.push((n, address)); - } - - // Then delete some, which is what makes this look like a real node rather than a - // fresh file. The environment is pinned to its current size for the whole migration, - // so a write during the bridge lands only if there are free pages to land in. On a - // production node there are plenty: this migration exists precisely because deleting - // millions of chunks filled the free list and returned nothing to the filesystem. - // Seeded and never deleted from, the environment would have no room and the bridge's - // second write would never happen, which is not the case worth testing. - let discarded: Vec<(usize, [u8; 32])> = keys.drain(..CHUNKS / 2).collect(); - for (_, address) in &discarded { - lmdb.delete(address).await.expect("make room"); - } - lmdb.wait_idle().await; - keys -} - -/// Child mode: retire the legacy environment, and be killed once it is marked. -/// -/// Everything before the mark is done here rather than in the parent, because the whole -/// point is that the process that wrote the mark is the one that dies. -#[tokio::test] -#[ignore = "child process of a crash test, not run on its own"] -async fn child_retires_until_killed() { - let root = child_root(); - let store = reopen(&root).await; - - let shutdown = tokio_util::sync::CancellationToken::new(); - let keys = store.legacy_only_keys(); - store - .copy_batch(&keys, 0, 0, &shutdown) - .await - .expect("copy every chunk"); - store.wait_idle().await; - store.commit_to_files().expect("commit to the file set"); - store.note_commitment_rebuilt(); - store.note_commitment_rebuilt(); - store.force_migration_state(|state| { - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map_or(0, |d| d.as_secs()); - state.committed_at_unix = Some( - now.saturating_sub(ant_node::storage::migration::MIN_RETIRE_DELAY_HOURS * 3600 + 60), - ); - }); - - let proof = store - .verify_before_retire(0, &shutdown) - .await - .expect("verify before retiring"); - // Parks inside this call, once the environment is renamed aside and marked. - let _ = store - .retire_legacy( - &proof, - &|_: &[u8; 32]| false, - &std::collections::BTreeSet::new(), - ) - .await; - panic!("the child finished retiring without being killed, so nothing was interrupted"); -} - -/// A process killed inside a publish leaves no chunk it cannot serve. -/// -/// The child is stopped at the last moment before the chunk's name exists on disk: on Unix -/// the bytes written to a temporary file with the rename not yet made, off Unix the point -/// before the file is created at all, since that platform writes under the final name -/// because a rename there carries no durability guarantee. The failure this guards against -/// is the same on both: a name outliving its bytes. The index is built from filenames at -/// startup, so a partial file wearing a real chunk name would be advertised, committed to, -/// and unservable. -#[tokio::test] -async fn a_process_killed_mid_publish_leaves_no_chunk_it_cannot_serve() { - let tmp = TempDir::new().expect("temp dir"); - let root = tmp.path().join("node"); - std::fs::create_dir_all(&root).expect("mkdir"); - - // Twenty chunks land before the crash, so the store this reopens has real content in - // it. Stopping the very first write would leave nothing indexed and the loop below - // would pass by iterating over nothing. - let marker = kill_child_at_failpoint( - "child_writes_until_killed", - &root, - ant_node::storage::file_store::HALT_BEFORE_PUBLISH, - 20, - ); - assert!( - marker.exists(), - "the child must have reached the failpoint before it was killed" - ); - - // Reopening is itself part of the assertion: a store that cannot start after a crash - // is a node that cannot start. - let store = reopen(&root).await; - for key in store.all_keys().await.expect("all_keys") { - let served = store.get(&key).await; - assert!( - matches!(served, Ok(Some(_))), - "chunk {} is claimed after a crash but cannot be served: {served:?}", - hex::encode(key) - ); - } -} - -/// The temporary file a killed publish left behind is swept, not indexed. -/// -/// It carries no chunk name, so it can never be served, and leaving it would cost disk -/// for the life of the node. -/// -/// Unix only, because the leftover only exists on Unix. Off Unix the store creates the -/// file under its final name and flushes it, deliberately, since a rename there is not -/// documented to be durable. So there is no temporary file to sweep and the equivalent -/// hazard is different: a real chunk name over bytes that are short or wrong. That one is -/// covered by the store's own tests, which run on every platform, and by the -/// re-hash-everything pass the retirement does before it deletes anything. -#[cfg(unix)] -#[tokio::test] -async fn the_leftovers_of_a_killed_publish_are_swept() { - let tmp = TempDir::new().expect("temp dir"); - let root = tmp.path().join("node"); - std::fs::create_dir_all(&root).expect("mkdir"); - - kill_child_at_failpoint( - "child_writes_until_killed", - &root, - ant_node::storage::file_store::HALT_BEFORE_PUBLISH, - 5, - ); - - let before = temp_files(&root.join("chunks")); - assert!( - before > 0, - "the child should have left a temporary file behind when it was killed" - ); - - let store = reopen(&root).await; - store.wait_idle().await; - assert_eq!( - temp_files(&root.join("chunks")), - 0, - "the store should sweep what an interrupted write left" - ); - drop(store); -} - -/// How many partly-written files are under `chunks_dir`. -#[cfg(unix)] -fn temp_files(chunks_dir: &Path) -> usize { - let Ok(shards) = std::fs::read_dir(chunks_dir) else { - return 0; - }; - shards - .flatten() - .filter_map(|shard| std::fs::read_dir(shard.path()).ok()) - .flat_map(std::iter::IntoIterator::into_iter) - .flatten() - .filter(|entry| { - entry - .file_name() - .to_str() - .is_some_and(|name| !name.chars().all(|c| c.is_ascii_hexdigit())) - }) - .count() -} - -/// Open the store the way a restart would. -async fn reopen(root: &Path) -> ChunkStore { - let mut config = ChunkStoreConfig { - root_dir: root.to_path_buf(), - disk_reserve: 0, - ..ChunkStoreConfig::default() - }; - config.migration.lock_dir = Some(root.to_path_buf()); - ChunkStore::new(config) - .await - .expect("the store must open after a crash") -} - -/// A crash part-way through copying loses nothing: the environment still has everything. -/// -/// The copier is only allowed to drop a key from its list once the file is durably -/// published, so a crash mid-copy costs the work of one chunk, never the chunk. -#[tokio::test] -async fn a_killed_migration_still_has_every_chunk_somewhere() { - let tmp = TempDir::new().expect("temp dir"); - let root = tmp.path().join("node"); - std::fs::create_dir_all(&root).expect("mkdir"); - let keys = seed_legacy_from(&root, 0).await; - - // Ten chunks copied, the eleventh interrupted. An earlier version killed the child - // after a fixed delay, which on a fast runner meant it had copied everything and on a - // slow one meant it had copied nothing; both make this test say something other than - // what it claims. - kill_child_at_failpoint( - "child_migrates_until_killed", - &root, - ant_node::storage::file_store::HALT_BEFORE_PUBLISH, - 10, - ); - - // Interrupted, which is two claims and not one: some chunks copied, and some not. - // Only the upper bound was checked before, so a copier that did nothing at all passed - // as long as everything was still readable from the environment. - let store = reopen(&root).await; - let left = store.legacy_only_keys().len(); - assert!( - left > 0, - "the child was supposed to be killed part-way through, not after finishing" - ); - assert!( - left < keys.len(), - "the child copied nothing, so nothing was interrupted: {left} of {} left", - keys.len() - ); - - for (n, key) in &keys { - let served = store - .get(key) - .await - .expect("read after a crash") - .expect("every seeded chunk must still be readable from one store or the other"); - assert_eq!(served, chunk_bytes(*n), "chunk {n} came back wrong"); - } -} - -/// A crash between the two halves of a dual write does not leave a chunk unprotected. -/// -/// The environment's copy is written first and the file second. A crash in between leaves -/// a chunk only the environment has, and it must be on the copier's list, because a key -/// in neither view is what retirement destroys. -/// -/// The chunks the child writes are deliberately ones the environment does not already -/// hold. An earlier version seeded the same addresses the child then wrote, and the write -/// path skips the environment half for a key that is already legacy-only, so no dual -/// write happened at all and the test proved nothing. -#[tokio::test] -async fn a_crash_between_the_two_halves_leaves_the_chunk_on_the_list() { - let tmp = TempDir::new().expect("temp dir"); - let root = tmp.path().join("node"); - std::fs::create_dir_all(&root).expect("mkdir"); - // Seeded with chunks the child will not write: the child starts at 0 and these are - // far above anything it reaches in the time it has. - seed_legacy_from(&root, 1_000_000).await; - - // The crash lands inside the write of chunk 20, so chunks 0 to 19 completed and 20 - // is the one caught between the two halves. - kill_child_at_failpoint( - "child_writes_until_killed", - &root, - ant_node::storage::file_store::HALT_BEFORE_PUBLISH, - 20, - ); - - let store = reopen(&root).await; - - // Whatever the environment holds and the file store does not is on the list. It is - // derived at open from the two key sets, which is the property that makes a crash - // survivable: re-read from disk, never carried across. - // The specific key, not merely a non-empty list. The environment was seeded with - // unrelated keys, and an earlier version asserted only that something was on the - // list, which those seeds satisfied whether or not a dual write had happened at all. - let interrupted = ant_node::client::compute_address(&chunk_bytes(20)); - let legacy_only = store.legacy_only_keys(); - assert!( - legacy_only.contains(&interrupted), - "the chunk whose file half never landed must be on the copier's list: it reached \ - the environment and nothing else knows about it" - ); - for key in &legacy_only { - let served = store - .get(key) - .await - .expect("read") - .expect("a key on the copier's list must be readable from the environment"); - assert_eq!(ant_node::client::compute_address(&served), *key); - } - - // And nothing the store claims is unservable, from either side of the union. - for key in store.all_keys().await.expect("all_keys") { - assert!( - matches!(store.get(&key).await, Ok(Some(_))), - "chunk {} is claimed after a crash but cannot be served", - hex::encode(key) - ); - } -} - -/// A retirement killed after the mark is finished on the next start, never reopened. -/// -/// The most destructive moment in the whole migration. By the time the mark is written the -/// environment has been renamed aside and the node has already told the network it serves -/// those chunks from the file store. A start that put the directory back would leave the -/// node running two stores again with the disk it came here to free still spent; a start -/// that deleted an *unmarked* directory would destroy a live environment. The mark is what -/// separates the two, and it is written by the process that then dies. -/// -/// Its recovery has unit tests that plant the mark by hand. What those cannot show is that -/// the production path really writes the mark at that moment, before anything is deleted -/// and by the process that then dies. That is what this settles. It does not settle -/// durability: killing a process keeps the kernel page cache, so surviving a kill is not -/// surviving a power cut, which stays a fleet gate. -#[tokio::test] -async fn a_retirement_killed_after_the_mark_is_finished_not_reopened() { - let tmp = TempDir::new().expect("temp dir"); - let root = tmp.path().join("node"); - std::fs::create_dir_all(&root).expect("mkdir"); - let seeded = seed_legacy_from(&root, 0).await; - - kill_child_at_failpoint( - "child_retires_until_killed", - &root, - ant_node::storage::file_store::HALT_AFTER_RETIRE_MARK, - 0, - ); - - // The child died with the directory renamed aside and marked. Nothing had been - // deleted, so this is the state a power cut would leave behind. - let store = reopen(&root).await; - for _ in 0..600 { - if !store.legacy_dir_is_on_disk() && tombstones(&root) == 0 { - break; - } - tokio::time::sleep(Duration::from_millis(100)).await; - } - assert!( - !store.legacy_dir_is_on_disk(), - "the marked environment was put back rather than finished" - ); - assert_eq!( - tombstones(&root), - 0, - "the marked directory is still on disk, so its space was never returned" - ); - - // And every chunk the environment held is still served, from the file store. - for (n, key) in &seeded { - let served = store - .get(key) - .await - .expect("read") - .expect("a chunk must survive an interrupted retirement"); - assert_eq!(served, chunk_bytes(*n), "chunk {n} came back wrong"); - } -} - -/// Directories beside the live environment that a retirement left behind. -fn tombstones(root: &Path) -> usize { - let Ok(entries) = std::fs::read_dir(root) else { - return 0; - }; - entries - .flatten() - .filter(|entry| { - entry.file_type().is_ok_and(|kind| kind.is_dir()) - && entry - .file_name() - .to_str() - .is_some_and(|name| name.starts_with(ant_node::storage::LEGACY_ENV_DIR)) - }) - .count() -} diff --git a/tests/migration_reclaims_disk.rs b/tests/migration_reclaims_disk.rs deleted file mode 100644 index 55cf07f3..00000000 --- a/tests/migration_reclaims_disk.rs +++ /dev/null @@ -1,390 +0,0 @@ -//! Proof that the migration actually returns disk to the filesystem. -//! -//! This is the claim the whole change exists to make good, and until now it was the one -//! thing the test suite did not check. The unit tests prove the environment is *removed*; -//! that is not the same as the space coming back, which is exactly the mistake that -//! started this work. The fleet deleted 2.29 million chunks, every counter said the -//! chunks were gone, and not one byte returned to the filesystem, because LMDB moves -//! freed pages to its own free list and never shortens the file. -//! -//! So these tests measure the filesystem, not the store's opinion of itself: the size of -//! the data on disk before and after, and the free space the operating system reports. -//! -//! They run on every platform CI covers, which is also the filesystem matrix that matters -//! here: ext4 on Linux, APFS on macOS, NTFS on Windows. - -#![allow( - clippy::unwrap_used, - clippy::expect_used, - clippy::panic, - clippy::missing_panics_doc, - // Test fixtures: every cast here is of a bounded loop counter into a byte, and the - // wrap is what makes the fill vary. - clippy::cast_possible_truncation -)] - -use ant_node::storage::migration::{MigrationPhase, MIN_RETIRE_DELAY_HOURS}; -use ant_node::storage::{ChunkStore, ChunkStoreConfig, LmdbStorage, LmdbStorageConfig}; -use std::path::Path; -use std::sync::Arc; -use tempfile::TempDir; -use tokio_util::sync::CancellationToken; - -/// Chunks to plant. Enough that the environment is meaningfully larger than its own -/// overhead, so "the file shrank" cannot be an accounting artefact. -const CHUNKS: usize = 400; - -/// Bytes per chunk. -/// -/// Sized against the noise, not against the chunk. This test reads what the *filesystem* -/// says is free, which on a shared runner moves for reasons that have nothing to do with -/// it: another job's build, a package cache, an indexer. At 16 KiB a chunk the whole -/// environment came to about 14 MB and the recovery threshold to about 7 MB, which -/// ordinary runner activity can swallow. At 128 KiB it is an order of magnitude clear of -/// that, and 400 chunks still write in a few seconds. -const CHUNK_BYTES: usize = 128 * 1024; - -/// Blocks actually allocated under `path`, in bytes, following no links. -/// -/// Allocated blocks rather than file lengths. A length is what the file claims; blocks -/// are what the filesystem has handed out, and the two part company exactly where this -/// test needs to be careful: a sparse file, a file whose last block is mostly padding, or -/// a file that has been unlinked while something still holds it open. -#[cfg(unix)] -fn allocated_bytes(path: &Path) -> u64 { - use std::os::unix::fs::MetadataExt; - walk(path, &|meta| meta.blocks() * 512) -} - -/// Off Unix, the length is the best the standard library offers. -#[cfg(not(unix))] -fn allocated_bytes(path: &Path) -> u64 { - walk(path, &|meta| meta.len()) -} - -/// Sum `size` over everything under `path`. -fn walk(path: &Path, size: &dyn Fn(&std::fs::Metadata) -> u64) -> u64 { - let Ok(entries) = std::fs::read_dir(path) else { - return std::fs::symlink_metadata(path).map_or(0, |m| size(&m)); - }; - entries - .flatten() - .map(|entry| { - let path = entry.path(); - match std::fs::symlink_metadata(&path) { - Ok(meta) if meta.is_dir() => walk(&path, size), - Ok(meta) => size(&meta), - Err(_) => 0, - } - }) - .sum() -} - -/// What the filesystem says is free, right now. -/// -/// The measurement that cannot be argued with, and the one this test exists for. A path -/// disappearing proves nothing: unlink a file that something still holds open and every -/// name is gone while every block is still spoken for, which is a fair description of the -/// bug that started all this. -fn free_space(path: &Path) -> u64 { - fs2::available_space(path).expect("the filesystem should report its free space") -} - -/// Deterministic content for chunk `n`, filled so it does not compress to nothing. -/// -/// `n` goes in verbatim at the front rather than being folded into the fill, because a -/// fill that wraps makes two different `n` produce the same bytes, and content-addressed -/// storage would then hold one chunk where the test believed it held two. The first -/// version of this test did exactly that and undercounted by a third. -fn chunk_bytes(n: usize) -> Vec { - let mut content = vec![0u8; CHUNK_BYTES]; - content[..8].copy_from_slice(&(n as u64).to_le_bytes()); - for (i, byte) in content.iter_mut().enumerate().skip(8) { - *byte = ((i.wrapping_mul(31)).wrapping_add(n) % 251) as u8; - } - content -} - -/// Plant a legacy environment holding `CHUNKS` chunks and close it. -async fn seed_legacy_environment(root: &Path) -> Vec<[u8; 32]> { - let lmdb = LmdbStorage::new(LmdbStorageConfig { - root_dir: root.to_path_buf(), - verify_on_read: true, - max_map_size: 0, - disk_reserve: 0, - }) - .await - .expect("open the legacy environment"); - - let mut keys = Vec::with_capacity(CHUNKS); - for n in 0..CHUNKS { - let content = chunk_bytes(n); - let address = ant_node::client::compute_address(&content); - lmdb.put(&address, &content).await.expect("seed a chunk"); - keys.push(address); - } - lmdb.wait_idle().await; - keys -} - -/// A store configured to migrate promptly, so a test does not wait out real delays. -fn migrating_config(root: &Path) -> ChunkStoreConfig { - let mut config = ChunkStoreConfig { - root_dir: root.to_path_buf(), - disk_reserve: 0, - ..ChunkStoreConfig::default() - }; - config.migration.tick_secs = 1; - config.migration.copier_throttle_mib_per_sec = 0; - config.migration.copier_slack_mb = 0; - config.migration.lock_dir = Some(root.to_path_buf()); - config -} - -/// Take a settled store all the way through retirement. -/// -/// Drives the steps the driver would, rather than running the driver, so the test does -/// not depend on wall-clock gates it has no business waiting for. The gates themselves -/// are covered by their own tests; what this one is about is the disk. -async fn copy_everything(store: &Arc, keys: &[[u8; 32]]) { - store - .copy_batch(keys, 0, 0, &CancellationToken::new()) - .await - .expect("copy every chunk into the file store"); - assert!( - store.legacy_only_keys().is_empty(), - "every chunk should have been copied" - ); - store.wait_idle().await; -} - -/// Take an already-copied store through retirement, returning the bytes it freed. -/// -/// Separate from the copying so a caller can measure the disk in between, at the peak -/// where both stores hold everything. That is the moment a node is most at risk of -/// filling up, and measuring only the ends would miss it. -async fn retire(store: &Arc) -> u64 { - let shutdown = CancellationToken::new(); - - store - .commit_to_files() - .expect("commit to the file-backed set"); - store.note_commitment_rebuilt(); - store.note_commitment_rebuilt(); - store.force_migration_state(|state| { - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map_or(0, |d| d.as_secs()); - state.committed_at_unix = Some(now.saturating_sub(MIN_RETIRE_DELAY_HOURS * 3600 + 60)); - }); - - let proof = store - .verify_before_retire(0, &shutdown) - .await - .expect("verify before retiring"); - assert!(proof.is_clean(), "verification must pass: {proof:?}"); - - store - .retire_legacy( - &proof, - &|_: &[u8; 32]| false, - &std::collections::BTreeSet::new(), - ) - .await - .expect("retire the legacy environment") -} - -/// The bytes the legacy environment occupied come back to the filesystem. -/// -/// Three measurements, because only the third one settles it: what the filesystem says is -/// free before anything is written, at the peak when both stores hold everything, and -/// after the environment is gone. A test that only watched paths disappear would pass -/// while every block stayed allocated, which is a fair description of the bug that -/// started all this. -/// -/// The numbers are noisy on a shared machine, so the assertion is about the shape: the -/// peak is materially below the start, and the end recovers most of the way back to it. -#[tokio::test] -async fn retiring_the_legacy_environment_returns_its_bytes_to_the_filesystem() { - let tmp = TempDir::new().expect("temp dir"); - let root = tmp.path().join("node"); - std::fs::create_dir_all(&root).expect("mkdir"); - - let payload = (CHUNKS * CHUNK_BYTES) as u64; - // What the reading drifts by here, with nothing of ours happening. Printed rather - // than asserted on: it is what tells whoever reads a failure whether the space did not - // come back or the machine was simply busy. - let quiet = free_space(&root); - tokio::time::sleep(std::time::Duration::from_millis(500)).await; - let drift = quiet.abs_diff(free_space(&root)); - - let free_at_start = free_space(&root); - - let keys = seed_legacy_environment(&root).await; - let environment = root.join("chunks.mdb"); - let environment_blocks = allocated_bytes(&environment); - assert!( - environment_blocks >= payload, - "the seeded environment should have at least the chunk bytes allocated, has \ - {environment_blocks}" - ); - - let store = Arc::new( - ChunkStore::new(migrating_config(&root)) - .await - .expect("open the store"), - ); - assert!(store.has_legacy()); - - // Both stores hold everything: the peak, and the moment a node is most at risk of - // filling its disk. - copy_everything(&store, &keys).await; - let free_at_peak = free_space(&root); - assert!( - free_at_peak < free_at_start, - "holding both copies should have consumed disk" - ); - - let freed = retire(&store).await; - store.wait_idle().await; - assert_eq!(store.migration_phase(), MigrationPhase::FilesOnly); - assert!(freed > 0, "retirement reported no bytes freed"); - - // The deletion runs on a detached thread so the node can serve while it happens. - for _ in 0..400 { - if !environment.exists() && allocated_bytes(&root) < environment_blocks + payload { - break; - } - tokio::time::sleep(std::time::Duration::from_millis(50)).await; - } - assert!( - !environment.exists(), - "the environment directory is still on disk" - ); - - // Dropping the store closes every handle. A file that is unlinked while something - // still holds it open keeps its blocks and shows in no directory, so measuring before - // this point would be measuring the wrong thing. - drop(store); - - // Polled rather than sampled once after a fixed pause. Not every filesystem updates - // its accounting the instant a file goes: btrfs in particular defers it, and a single - // reading taken too early says the space never came back when it is on its way. The - // deadline is what makes this a test rather than a wait. - let free_at_end = wait_for_space(&root, free_at_peak + environment_blocks / 2).await; - - // Only the file store's copy should be left. - let left_on_disk = allocated_bytes(&root); - let file_store_blocks = allocated_bytes(&root.join("chunks")); - let recovered = free_at_end.saturating_sub(free_at_peak); - assert!( - left_on_disk <= file_store_blocks + (payload / 10), - "something other than the file store is still using disk: {left_on_disk} total \ - against {file_store_blocks} in the file store" - ); - - // And the filesystem agrees, measured against the peak rather than against a guess. - // Retiring should hand back most of what the environment was occupying, which makes - // this a statement about the environment's own size rather than about the payload. - assert!( - recovered > environment_blocks / 2, - "retiring recovered {recovered} bytes of an environment occupying \ - {environment_blocks}" - ); - - // And what is left costs roughly one copy rather than two, measured against what the - // file store actually occupies rather than against what the environment did. The two - // are not interchangeable: how much a filesystem spends on four hundred small files - // against one large one is its own business, and btrfs in particular charges very - // differently for the two. Printed as well as asserted, so a number that is drifting - // shows up in the log before it trips anything. - let consumed = free_at_start.saturating_sub(free_at_end); - println!( - "reclaim: environment {environment_blocks} bytes, file store {file_store_blocks}, \ - peak cost {}, end cost {consumed}, recovered {recovered}, ambient drift {drift} \ - in half a second", - free_at_start.saturating_sub(free_at_peak) - ); - assert!( - consumed < file_store_blocks + environment_blocks / 2, - "the filesystem is still down {consumed} bytes with only {file_store_blocks} of \ - file store to account for it, so the environment's space did not come back" - ); - - // Every chunk is still served, read back through a store opened from scratch, which - // is what a restart does. Space recovered by losing data would be no achievement, and - // it is the failure this whole change exists to avoid. - let fresh = store_reopened(&root).await; - for (n, key) in keys.iter().enumerate() { - let served = fresh - .get(key) - .await - .expect("read a migrated chunk") - .expect("a migrated chunk should still be there"); - assert_eq!(served, chunk_bytes(n), "chunk {n} came back wrong"); - } -} - -/// Wait for the filesystem to report at least `wanted` bytes free, and return what it -/// reports at the end. -/// -/// Returns whatever it last saw when the deadline passes, so the caller's assertion is -/// what fails rather than this helper, and the number in the failure is a real reading. -async fn wait_for_space(path: &Path, wanted: u64) -> u64 { - let deadline = std::time::Instant::now() + std::time::Duration::from_mins(1); - loop { - let free = free_space(path); - if free >= wanted || std::time::Instant::now() > deadline { - return free; - } - tokio::time::sleep(std::time::Duration::from_millis(100)).await; - } -} - -/// Open the store again from scratch, which is what a restart does. -async fn store_reopened(root: &Path) -> ChunkStore { - ChunkStore::new(ChunkStoreConfig { - root_dir: root.to_path_buf(), - disk_reserve: 0, - ..ChunkStoreConfig::default() - }) - .await - .expect("the store must reopen after retirement") -} - -/// The file store holds the same payload in less space than the environment did. -/// -/// Not a compression claim: it is that one file per chunk carries no free list and no -/// map overhead, which is the whole reason the space can be returned at all. -#[tokio::test] -async fn the_file_store_holds_the_same_chunks_in_less_space() { - let tmp = TempDir::new().expect("temp dir"); - let root = tmp.path().join("node"); - std::fs::create_dir_all(&root).expect("mkdir"); - - let keys = seed_legacy_environment(&root).await; - let environment_bytes = allocated_bytes(&root.join("chunks.mdb")); - - let store = Arc::new( - ChunkStore::new(migrating_config(&root)) - .await - .expect("open the store"), - ); - store - .copy_batch(&keys, 0, 0, &CancellationToken::new()) - .await - .expect("copy"); - store.wait_idle().await; - - let payload = (CHUNKS * CHUNK_BYTES) as u64; - let file_store_bytes = allocated_bytes(&root.join("chunks")); - assert!( - file_store_bytes >= payload, - "the file store should hold at least the payload: {file_store_bytes} < {payload}" - ); - assert!( - file_store_bytes <= environment_bytes, - "one file per chunk should not cost more than the environment did: \ - {file_store_bytes} > {environment_bytes}" - ); -} diff --git a/tests/migration_shared_volume.rs b/tests/migration_shared_volume.rs deleted file mode 100644 index 8ff79791..00000000 --- a/tests/migration_shared_volume.rs +++ /dev/null @@ -1,512 +0,0 @@ -//! Several nodes migrating on one disk. -//! -//! Operators run many nodes per machine, and during the bridge each one briefly holds two -//! copies of everything it stores. If they all did that at once the disk would fill, which -//! is the failure this migration exists to prevent rather than cause. A lock keyed by the -//! filesystem lets one node at a time do the copying. -//! -//! The lock has a cap on how long a single node may hold it, so one node stuck waiting on -//! its neighbours cannot keep the rest of the machine from ever starting. That cap is -//! hours long by design, so what is checked here is the exclusion itself and the -//! accounting around it, not the cap expiring. - -#![allow( - clippy::unwrap_used, - clippy::expect_used, - clippy::panic, - clippy::missing_panics_doc, - // Test fixtures: every cast here is of a bounded loop counter into a byte, and the - // wrap is what makes the fill vary. - clippy::cast_possible_truncation -)] - -use ant_node::storage::migration::{self, LockAttempt, VolumeLock, MIN_RETIRE_DELAY_HOURS}; -use ant_node::storage::{ChunkStore, ChunkStoreConfig, LmdbStorage, LmdbStorageConfig}; -use std::path::Path; -use std::sync::Arc; -use std::time::Duration; -use tempfile::TempDir; -use tokio_util::sync::CancellationToken; - -/// Chunks each node holds. -const CHUNKS: usize = 60; - -fn chunk_bytes(node: usize, n: usize) -> Vec { - let mut content = vec![0u8; 8192]; - content[..8].copy_from_slice(&(n as u64).to_le_bytes()); - content[8..16].copy_from_slice(&(node as u64).to_le_bytes()); - for (i, byte) in content.iter_mut().enumerate().skip(16) { - *byte = ((i.wrapping_mul(29)).wrapping_add(n).wrapping_add(node) % 251) as u8; - } - content -} - -async fn seed_legacy(root: &Path, node: usize) -> Vec<[u8; 32]> { - let lmdb = LmdbStorage::new(LmdbStorageConfig { - root_dir: root.to_path_buf(), - verify_on_read: true, - max_map_size: 0, - disk_reserve: 0, - }) - .await - .expect("open legacy"); - let mut keys = Vec::new(); - for n in 0..CHUNKS { - let content = chunk_bytes(node, n); - let address = ant_node::client::compute_address(&content); - lmdb.put(&address, &content).await.expect("seed"); - keys.push(address); - } - lmdb.wait_idle().await; - keys -} - -/// One node at a time copies; the others wait rather than piling on. -/// -/// The lock is taken per filesystem, not per node directory. That distinction is the -/// whole point: two nodes are configured with different roots by definition, so a lock -/// beside each root would serialise neither against the other. -#[test] -fn only_one_node_on_a_volume_holds_the_lock() { - let volume = TempDir::new().expect("temp dir"); - let node_a = volume.path().join("node-a"); - let node_b = volume.path().join("node-b"); - let node_c = volume.path().join("node-c"); - for root in [&node_a, &node_b, &node_c] { - std::fs::create_dir_all(root).expect("mkdir"); - } - - // Scoped to this volume directory so the test does not contend with anything else on - // the machine's real filesystem. - let scope = Some(volume.path()); - - let LockAttempt::Acquired(held) = VolumeLock::try_acquire(&node_a, scope) else { - panic!("the first node must take the lock"); - }; - assert!( - matches!(VolumeLock::try_acquire(&node_b, scope), LockAttempt::Busy), - "a second node on the same volume must wait" - ); - assert!( - matches!(VolumeLock::try_acquire(&node_c, scope), LockAttempt::Busy), - "and so must a third" - ); - - drop(held); - assert!( - matches!( - VolumeLock::try_acquire(&node_b, scope), - LockAttempt::Acquired(_) - ), - "the lock must pass on once the first node lets go" - ); -} - -/// A migration context with no network, which is all these tests need. -/// -/// The gates that consult routing have their own tests; what is under test here is the -/// lock, and a node with no view of the network still copies. -fn offline_context() -> migration::MigrationContext { - migration::MigrationContext { - p2p: None, - self_id: None, - self_xor: None, - commitment: None, - replication: None, - sync_state: None, - audit_challenge_coordinator: None, - peer_commitments: None, - close_group_size: 7, - } -} - -/// Two migration drivers on one disk: only one copies at a time. -/// -/// This drives `migration::run`, not `copy_batch`. The copier does not take the volume -/// lock; the driver does, and an earlier version of this test called the copier directly -/// and would have passed with the lock removed from the driver entirely. -#[tokio::test] -async fn two_drivers_on_one_volume_do_not_copy_at_the_same_time() { - let volume = TempDir::new().expect("temp dir"); - let mut stores = Vec::new(); - let mut all_keys = Vec::new(); - - for node in 0..2 { - let root = volume.path().join(format!("node-{node}")); - std::fs::create_dir_all(&root).expect("mkdir"); - let keys = seed_legacy(&root, node).await; - - stores.push(driven_node(volume.path(), &root).await); - all_keys.push(keys); - } - - // Hold the volume before either driver starts, so both are shut out and neither can - // be observed making progress. - let LockAttempt::Acquired(held) = - VolumeLock::try_acquire(&volume.path().join("an-outsider"), Some(volume.path())) - else { - panic!("the outsider must take the lock"); - }; - - let shutdown = CancellationToken::new(); - let drivers: Vec<_> = stores - .iter() - .map(|store| { - tokio::spawn(migration::run( - Arc::clone(store), - offline_context(), - shutdown.clone(), - )) - }) - .collect(); - - // Long enough for several ticks. Neither driver may copy anything while the lock is - // held by somebody else. - tokio::time::sleep(Duration::from_secs(4)).await; - for (node, store) in stores.iter().enumerate() { - assert_eq!( - store.legacy_only_keys().len(), - CHUNKS, - "node {node} copied while another holder had the volume" - ); - } - - // Released: one of them takes it and copies. The other must not, because the holder - // keeps the volume from its first copy through to retiring, rather than handing it - // back between chunks. That is the point of the lock: two nodes copying at once each - // hold two copies of everything, and the disk this migration exists to free is the - // one that fills. - drop(held); - let mut copier = None; - for _ in 0..300 { - if let Some(node) = stores - .iter() - .position(|s| s.legacy_only_keys().len() < CHUNKS) - { - copier = Some(node); - break; - } - tokio::time::sleep(Duration::from_millis(100)).await; - } - let copier = copier.expect("one driver should have taken the volume and started"); - - // Give the other one many ticks to misbehave in. - tokio::time::sleep(Duration::from_secs(3)).await; - let waiting = 1 - copier; - assert_eq!( - stores[waiting].legacy_only_keys().len(), - CHUNKS, - "node {waiting} copied while node {copier} held the volume" - ); - - // And the one that has it finishes copying, checking on every tick that the other has - // still not started. The window being watched is the whole of the first node's copy - // rather than its two ends. - // - // Copying is as far as this one goes. Retirement is gated behind hours of wall clock - // that a test has no business waiting out, so whether the lock spans that half too has - // its own test below. - for _ in 0..600 { - assert_eq!( - stores[waiting].legacy_only_keys().len(), - CHUNKS, - "node {waiting} copied while node {copier} still held the volume" - ); - if stores[copier].legacy_only_keys().is_empty() { - break; - } - tokio::time::sleep(Duration::from_millis(100)).await; - } - shutdown.cancel(); - for driver in drivers { - let _ = driver.await; - } - - assert!( - stores[copier].legacy_only_keys().is_empty(), - "the node holding the volume did not finish copying" - ); - // Both of them, not just the one that went first. Counting only the copier would pass - // for a node that had picked up its neighbour's chunks as well as its own. - for (node, store) in stores.iter().enumerate() { - holds_exactly_its_own(store, node, &all_keys[node]).await; - } -} - -/// A node set up to be driven by `migration::run` on a shared volume. -async fn driven_node(volume: &Path, root: &Path) -> Arc { - let mut config = ChunkStoreConfig { - root_dir: root.to_path_buf(), - disk_reserve: 0, - ..ChunkStoreConfig::default() - }; - config.migration.tick_secs = 1; - config.migration.copier_throttle_mib_per_sec = 0; - config.migration.copier_slack_mb = 0; - // Small enough that copying takes several ticks, so there is a window in which the - // other node could misbehave and be caught, and large enough that the whole thing - // finishes in seconds rather than one chunk per tick. - config.migration.batch_chunks = 8; - config.migration.lock_dir = Some(volume.to_path_buf()); - Arc::new(ChunkStore::new(config).await.expect("open a node")) -} - -/// Every chunk this node seeded is still served, and nothing else is. -async fn holds_exactly_its_own(store: &ChunkStore, node: usize, keys: &[[u8; 32]]) { - assert_eq!( - store.current_chunks().expect("count") as usize, - keys.len(), - "node {node} must hold its own chunks and only its own" - ); - for (n, key) in keys.iter().enumerate() { - let served = store - .get(key) - .await - .expect("read") - .expect("every chunk this node seeded must still be here"); - assert_eq!( - served, - chunk_bytes(node, n), - "node {node} chunk {n} is wrong" - ); - } -} - -/// A node waits for the volume before it retires, not only before it copies. -/// -/// The driver is documented as holding the volume from the first copy through retirement -/// and not handing it back in between. The test above covers the copying half. This one -/// covers the other, which is the half that matters most: retiring means re-reading every -/// chunk in the store to verify it and then deleting an environment, so it is the heaviest -/// the disk gets. A driver that took the lock only for copying would run that pass while -/// eleven neighbours ran theirs. -/// -/// Shaped the same way as the copying test, and for the same reason: an outsider holds the -/// volume first, so the answer does not depend on catching a short window. The node is put -/// in the phase where retirement is the next thing it would do, and then watched for not -/// doing it. -#[tokio::test] -async fn a_node_waits_for_the_volume_before_it_retires() { - let volume = TempDir::new().expect("temp dir"); - let root = volume.path().join("node-0"); - std::fs::create_dir_all(&root).expect("mkdir"); - let keys = seed_legacy(&root, 0).await; - - let store = ready_to_retire(volume.path(), &root, &keys).await; - let shutdown = CancellationToken::new(); - - let LockAttempt::Acquired(held) = - VolumeLock::try_acquire(&volume.path().join("an-outsider"), Some(volume.path())) - else { - panic!("the outsider must take the lock"); - }; - - let driver = tokio::spawn(migration::run( - Arc::clone(&store), - offline_context(), - shutdown.clone(), - )); - - // Several ticks with everything else in place. The environment must still be there. - for _ in 0..40 { - assert!( - store.legacy_dir_is_on_disk(), - "the node retired while an outsider held the volume" - ); - tokio::time::sleep(Duration::from_millis(100)).await; - } - - // Released: now it retires. Without this half the test would pass against a node that - // never retires at all, which is the failure the whole migration exists to avoid. - drop(held); - let mut retired = false; - for _ in 0..600 { - if !store.legacy_dir_is_on_disk() && !store.has_legacy() { - retired = true; - break; - } - tokio::time::sleep(Duration::from_millis(100)).await; - } - shutdown.cancel(); - let _ = driver.await; - assert!( - retired, - "the node never retired once the volume was free, so it was not waiting for it" - ); - - // And it still holds everything it had. Retiring is deleting the old copy, not the - // chunks. - for (n, key) in keys.iter().enumerate() { - let served = store - .get(key) - .await - .expect("read") - .expect("every chunk must survive retirement"); - assert_eq!( - served, - chunk_bytes(0, n), - "chunk {n} is wrong after retiring" - ); - } -} - -/// Open a node whose only remaining migration work is to retire. -/// -/// Copied and committed by hand rather than by waiting for the driver, because the driver -/// gets here by waiting out the shed hold, which is days. Those gates have their own -/// tests; what the caller is about to watch is the volume lock. -async fn ready_to_retire(volume: &Path, root: &Path, keys: &[[u8; 32]]) -> Arc { - let mut config = ChunkStoreConfig { - root_dir: root.to_path_buf(), - disk_reserve: 0, - ..ChunkStoreConfig::default() - }; - config.migration.tick_secs = 1; - config.migration.copier_throttle_mib_per_sec = 0; - config.migration.copier_slack_mb = 0; - config.migration.lock_dir = Some(volume.to_path_buf()); - let store = Arc::new(ChunkStore::new(config).await.expect("open a node")); - - store - .copy_batch(keys, 0, 0, &CancellationToken::new()) - .await - .expect("copy every chunk"); - store.wait_idle().await; - store.commit_to_files().expect("commit to the file set"); - store.note_commitment_rebuilt(); - store.note_commitment_rebuilt(); - store.force_migration_state(|state| { - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map_or(0, |d| d.as_secs()); - // Past the delay that buys the rollback window, so the only thing left between - // this node and deleting its environment is the volume. - state.committed_at_unix = Some(now.saturating_sub(MIN_RETIRE_DELAY_HOURS * 3600 + 60)); - }); - assert!( - store.legacy_dir_is_on_disk(), - "the environment should still be here before the driver runs" - ); - store -} - -/// Two nodes sharing a disk both finish, and neither loses a chunk to the other. -/// -/// Run one after the other, which is what the lock produces. What is checked is that the -/// second node's copy is unaffected by the first having already run on the same -/// filesystem: no shared state, no name collisions, no lock left behind. -/// -/// Copying only. It drives `copy_batch` rather than the driver, so it says nothing about -/// retirement and is not named as if it did: disabling retirement altogether would leave -/// it green. Retirement on a shared volume is -/// [`a_node_waits_for_the_volume_before_it_retires`]. -#[tokio::test] -async fn nodes_sharing_a_volume_do_not_take_each_others_chunks() { - let volume = TempDir::new().expect("temp dir"); - let shutdown = CancellationToken::new(); - - let mut nodes = Vec::new(); - for node in 0..2 { - let root = volume.path().join(format!("node-{node}")); - std::fs::create_dir_all(&root).expect("mkdir"); - let keys = seed_legacy(&root, node).await; - nodes.push((root, keys)); - } - - for (node, (root, keys)) in nodes.iter().enumerate() { - let mut config = ChunkStoreConfig { - root_dir: root.clone(), - disk_reserve: 0, - ..ChunkStoreConfig::default() - }; - config.migration.lock_dir = Some(volume.path().to_path_buf()); - let store = ChunkStore::new(config).await.expect("open"); - - store.copy_batch(keys, 0, 0, &shutdown).await.expect("copy"); - store.wait_idle().await; - - assert!( - store.legacy_only_keys().is_empty(), - "node {node} should have copied everything" - ); - for (n, key) in keys.iter().enumerate() { - let served = store - .get(key) - .await - .expect("read") - .expect("every chunk this node seeded must still be here"); - assert_eq!( - served, - chunk_bytes(node, n), - "node {node} chunk {n} came back wrong" - ); - } - } - - // Neither node picked up the other's chunks, which sharing a filesystem must not - // cause: the stores are separate, only the lock is shared. - let (root_a, keys_a) = &nodes[0]; - let store_a = ChunkStore::new(ChunkStoreConfig { - root_dir: root_a.clone(), - disk_reserve: 0, - ..ChunkStoreConfig::default() - }) - .await - .expect("reopen node 0"); - assert_eq!( - store_a.current_chunks().expect("count") as usize, - keys_a.len(), - "a node must hold its own chunks and only its own" - ); -} - -/// A node that cannot take the lock does not migrate, and does not lose anything either. -/// -/// Waiting is the correct answer: the chunks stay where they are, served from both stores, -/// until the volume is free. -#[tokio::test] -async fn a_node_that_cannot_take_the_lock_keeps_serving() { - let volume = TempDir::new().expect("temp dir"); - let root = volume.path().join("waiting-node"); - std::fs::create_dir_all(&root).expect("mkdir"); - let keys = seed_legacy(&root, 0).await; - - let LockAttempt::Acquired(_held) = - VolumeLock::try_acquire(&volume.path().join("busy-node"), Some(volume.path())) - else { - panic!("the other node must take the lock"); - }; - - let store = driven_node(volume.path(), &root).await; - - // A real driver, running the whole time. Without one this would say only that a store - // opens, and would still pass against a driver that ignored the lock entirely. - let shutdown = CancellationToken::new(); - let driver = tokio::spawn(migration::run( - Arc::clone(&store), - offline_context(), - shutdown.clone(), - )); - - // The store opens and serves regardless of the lock: only the copier waits for it. - assert!(store.has_legacy()); - for _ in 0..30 { - assert_eq!( - store.legacy_only_keys().len(), - CHUNKS, - "the node copied while another held the volume" - ); - for (n, key) in keys.iter().enumerate() { - let served = store - .get(key) - .await - .expect("read") - .expect("a node waiting for the volume still serves everything it holds"); - assert_eq!(served, chunk_bytes(0, n)); - } - tokio::time::sleep(Duration::from_millis(100)).await; - } - - shutdown.cancel(); - let _ = driver.await; -} diff --git a/tests/poc_audit_handler_live.rs b/tests/poc_audit_handler_live.rs index 9200e62b..1fb958d7 100644 --- a/tests/poc_audit_handler_live.rs +++ b/tests/poc_audit_handler_live.rs @@ -360,7 +360,7 @@ async fn committed_key_with_missing_bytes_is_rejected() { /// A successful proof reports what it read and hashed, at a floor per leaf. /// Anchors the rejection case below: it fixes what the measurement means. /// -/// A leaf costs more than its bytes — an LMDB lookup and a blocking-task round +/// A leaf costs more than its bytes — a store lookup and a blocking-task round /// trip are owed whatever its size — and nothing bounds a chunk from below, so /// the charge is `max(content, floor)` per leaf. These test records are 1 KiB, /// well under the floor, which is the case that used to be nearly free: the diff --git a/tests/poc_shutdown_lmdb_drain.rs b/tests/poc_shutdown_lmdb_drain.rs deleted file mode 100644 index 135e8001..00000000 --- a/tests/poc_shutdown_lmdb_drain.rs +++ /dev/null @@ -1,192 +0,0 @@ -//! Regression test for the LMDB drain guarantee of -//! [`ant_node::ReplicationEngine::shutdown`]. -//! -//! ## The vulnerability (pre-fix) -//! -//! Engine tasks race their work against the shutdown `CancellationToken` in -//! `select!`. Dropping the losing future does **not** cancel a -//! `tokio::task::spawn_blocking` LMDB transaction it was awaiting — the -//! closure keeps running on the blocking pool and owns a cloned heed `Env`. -//! `shutdown()` had nothing to wait on for those detached closures (fetch -//! `storage.put`, prune `storage.delete` / `paid_list.remove_batch`, -//! verification `paid_list.insert`), so it could return while the -//! environment was still open. Reopening the same LMDB file with the old -//! `Env` alive in-process is undefined behavior. -//! -//! ## The fix -//! -//! `ChunkStore` (via its file store) and `PaidList` track their blocking tasks in a -//! `TaskTracker`; `shutdown()` awaits `wait_idle()` on both after draining -//! its own tasks. This test parks a chunk-store write inside its blocking -//! closure, drops the awaiter (the exact leak shape), and asserts that -//! `shutdown()` blocks until the write finishes — then proves both LMDB -//! environments reopen cleanly. - -#![allow( - clippy::unwrap_used, - clippy::expect_used, - clippy::panic, - clippy::missing_panics_doc -)] - -use ant_node::payment::{ - EvmVerifierConfig, PaymentVerifier, PaymentVerifierConfig, PriceFloorConfig, -}; -use ant_node::replication::paid_list::PaidList; -use ant_node::storage::{ChunkStore, ChunkStoreConfig}; -use ant_node::{ReplicationConfig, ReplicationEngine}; -use evmlib::{Network as EvmNetwork, RewardsAddress}; -use rand::Rng; -use saorsa_core::identity::NodeIdentity; -use saorsa_core::{NodeConfig as CoreNodeConfig, P2PNode}; -use std::sync::Arc; -use std::time::Duration; -use tokio_util::sync::CancellationToken; - -/// E2E test port range (CLAUDE.md): tests must stay inside 20000-60000, -/// away from production ant-node's 10000-10999. -const TEST_PORT_RANGE_MIN: u16 = 20_000; -/// Upper bound (exclusive) of the E2E test port range. -const TEST_PORT_RANGE_MAX: u16 = 60_000; -/// Attempts to bind a random test port before giving up (mirrors the -/// transient port-bind retry in the e2e testnet harness). -const PORT_BIND_ATTEMPTS: usize = 4; -/// Short probe proving `shutdown()` is still waiting on the parked LMDB op. -const SHUTDOWN_BLOCKED_PROBE: Duration = Duration::from_millis(300); -/// Generous ceiling for `shutdown()` to finish once the op is released. -const SHUTDOWN_COMPLETE_TIMEOUT: Duration = Duration::from_secs(30); -/// Payment cache capacity for the test verifier. -const TEST_PAYMENT_CACHE_CAPACITY: usize = 1000; -/// Rewards address for the test verifier. -const TEST_REWARDS_ADDRESS: [u8; 20] = [0x01; 20]; - -/// Create and start a loopback P2P node on a random port in the test range. -async fn start_p2p_node(identity: &Arc) -> Arc { - let mut last_err = String::new(); - for _ in 0..PORT_BIND_ATTEMPTS { - let port = rand::thread_rng().gen_range(TEST_PORT_RANGE_MIN..TEST_PORT_RANGE_MAX); - let mut config = CoreNodeConfig::builder() - .port(port) - .ipv6(false) - .local(true) - .build() - .expect("build core config"); - config.node_identity = Some(Arc::clone(identity)); - match P2PNode::new(config).await { - Ok(node) => { - node.start().await.expect("start p2p node"); - return Arc::new(node); - } - Err(e) => last_err = e.to_string(), - } - } - panic!("failed to create P2P node after {PORT_BIND_ATTEMPTS} attempts: {last_err}"); -} - -/// A blocking LMDB write whose awaiter was dropped must delay `shutdown()` -/// until it commits, after which both LMDB environments reopen cleanly. -// Holding the gate's write guard across awaits is the point of the test: -// it parks the blocking closure while we probe shutdown(). -#[allow(clippy::await_holding_lock)] -#[tokio::test] -async fn shutdown_waits_for_detached_lmdb_op_and_envs_reopen() { - let temp_dir = tempfile::TempDir::new().expect("create temp dir"); - let root_dir = temp_dir.path().to_path_buf(); - - // The chunk store the engine will hold (and whose env we reopen below). - let storage = Arc::new( - ChunkStore::new(ChunkStoreConfig { - root_dir: root_dir.clone(), - ..ChunkStoreConfig::test_default() - }) - .await - .expect("create storage"), - ); - - let identity = Arc::new(NodeIdentity::generate().expect("generate identity")); - let replication_config = ReplicationConfig::default(); - let payment_verifier = Arc::new(PaymentVerifier::new(PaymentVerifierConfig { - evm: EvmVerifierConfig { - network: EvmNetwork::ArbitrumSepoliaTest, - }, - cache_capacity: TEST_PAYMENT_CACHE_CAPACITY, - close_group_size: replication_config.close_group_size, - local_rewards_address: RewardsAddress::new(TEST_REWARDS_ADDRESS), - price_floor: PriceFloorConfig::default(), - })); - - let p2p = start_p2p_node(&identity).await; - - let (_fresh_tx, fresh_rx) = tokio::sync::mpsc::unbounded_channel(); - let mut engine = ReplicationEngine::new( - replication_config, - Arc::clone(&p2p), - Arc::clone(&storage), - payment_verifier, - identity, - &root_dir, - fresh_rx, - CancellationToken::new(), - ) - .await - .expect("create engine"); - engine.start(p2p.dht_manager().subscribe_events()); - - // Park a put's blocking closure on the test gate, then drop its awaiter - // mid-flight — the exact shape of a select! losing to the shutdown token - // while `storage.put()` awaits `spawn_blocking`. - let content = b"held-open write must block engine shutdown"; - let address = ChunkStore::compute_address(content); - let gate = storage.test_put_gate(); - let parked = gate.write(); - tokio::select! { - biased; - res = storage.put(&address, content) => { - panic!("put must be parked on the test gate, got {res:?}") - } - () = std::future::ready(()) => {} - } - - { - let shutdown_fut = engine.shutdown(); - tokio::pin!(shutdown_fut); - - // shutdown() must not return while the blocking op is still running. - let blocked = tokio::time::timeout(SHUTDOWN_BLOCKED_PROBE, shutdown_fut.as_mut()).await; - assert!( - blocked.is_err(), - "shutdown() returned while a store write was in flight" - ); - - // Release the write; shutdown must now run to completion. - drop(parked); - tokio::time::timeout(SHUTDOWN_COMPLETE_TIMEOUT, shutdown_fut) - .await - .expect("shutdown after releasing the parked op"); - } - - // The detached write committed before shutdown returned. - assert!(storage.exists(&address).expect("exists after shutdown")); - - // Release every reference the test still holds. Per the shutdown - // contract, no engine-spawned work holds the storage or paid list any - // more, so these drops close both environments. - drop(engine); - p2p.shutdown().await.expect("p2p shutdown"); - drop(p2p); - drop(gate); - drop(storage); - - // Both LMDB environments reopen cleanly from the same directory. - let reopened = ChunkStore::new(ChunkStoreConfig { - root_dir: root_dir.clone(), - ..ChunkStoreConfig::test_default() - }) - .await - .expect("reopen chunk store"); - let read_back = reopened.get(&address).await.expect("get after reopen"); - assert_eq!(read_back, Some(content.to_vec())); - - let paid_list = PaidList::new(&root_dir).await.expect("reopen paid list"); - assert_eq!(paid_list.count().expect("paid list count"), 0); -} diff --git a/tests/storage_scale.rs b/tests/storage_scale.rs index 656eedf1..d5ac2627 100644 --- a/tests/storage_scale.rs +++ b/tests/storage_scale.rs @@ -24,7 +24,7 @@ clippy::cast_possible_truncation )] -use ant_node::storage::{FileStore, FileStoreConfig}; +use ant_node::storage::{ChunkStore, ChunkStoreConfig}; use std::path::Path; use std::time::{Duration, Instant}; use tempfile::TempDir; @@ -149,7 +149,7 @@ async fn opening_a_large_store_stays_quick() { let before = resident_bytes(); let opening = Instant::now(); - let store = FileStore::new(FileStoreConfig { + let store = ChunkStore::new(ChunkStoreConfig { root_dir: root.clone(), verify_on_read: true, disk_reserve: 0, @@ -229,7 +229,7 @@ async fn opening_a_large_store_stays_quick() { /// Open a store at `root`, time the scan, and close it again. async fn time_a_scan(root: &Path) -> Duration { let started = Instant::now(); - let store = FileStore::new(FileStoreConfig { + let store = ChunkStore::new(ChunkStoreConfig { root_dir: root.to_path_buf(), verify_on_read: true, disk_reserve: 0, @@ -328,7 +328,7 @@ async fn child_reports_index_memory() { let keys = key_count(); let before = resident_bytes().expect("linux reports this"); - let store = FileStore::new(FileStoreConfig { + let store = ChunkStore::new(ChunkStoreConfig { root_dir: root, verify_on_read: true, disk_reserve: 0, @@ -360,7 +360,7 @@ async fn each_chunk_the_store_writes_costs_one_directory_entry() { let root = tmp.path().join("node"); std::fs::create_dir_all(&root).expect("mkdir"); - let store = FileStore::new(FileStoreConfig { + let store = ChunkStore::new(ChunkStoreConfig { root_dir: root.clone(), verify_on_read: true, disk_reserve: 0, @@ -415,7 +415,7 @@ fn count_entries(path: &Path) -> Entries { // layout marker, and the lock that keeps a second process out. Both are // fixed, so neither grows with the store. Ok(_) - if entry.file_name() == ant_node::storage::file_store::LAYOUT_FILE_NAME + if entry.file_name() == ant_node::storage::chunk_store::LAYOUT_FILE_NAME || entry.file_name() == ".lock" => {} Ok(_) => counted.files += 1, Err(_) => {} @@ -447,7 +447,7 @@ async fn the_startup_scan_does_not_read_chunk_contents() { plant_sized(&root.join("chunks"), keys, chunk); let before = bytes_read().expect("linux reports this"); - let store = FileStore::new(FileStoreConfig { + let store = ChunkStore::new(ChunkStoreConfig { root_dir: root.clone(), verify_on_read: true, disk_reserve: 0,